luo980/curl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci/tests: Send test results to Azure DevOps for reporting

Browse Source
This commit is contained in:
Marc Hoersken 2020-02-22 20:09:29 +01:00
parent fa0216b294
commit ada581f2cc
No known key found for this signature in database
GPG Key ID: 61E03CBED7BC859E
3 changed files with 206 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.azure-pipelines.yml
View File

@ -29,6 +29,8 @@ stages:
- script: make test-nonflaky - script: make test-nonflaky
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: disable_ipv6 - job: disable_ipv6
displayName: ubuntu w/o IPv6 displayName: ubuntu w/o IPv6
@ -43,6 +45,8 @@ stages:
- script: make test-nonflaky - script: make test-nonflaky
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: disable_http_smtp_imap - job: disable_http_smtp_imap
displayName: ubuntu w/o HTTP/SMTP/IMAP displayName: ubuntu w/o HTTP/SMTP/IMAP
@ -57,6 +61,8 @@ stages:
- script: make test-nonflaky - script: make test-nonflaky
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: disable_thredres - job: disable_thredres
displayName: ubuntu sync resolver displayName: ubuntu sync resolver
@ -71,6 +77,8 @@ stages:
- script: make test-nonflaky - script: make test-nonflaky
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: http_only - job: http_only
displayName: ubuntu HTTP only displayName: ubuntu HTTP only
@ -85,6 +93,8 @@ stages:
- script: make test-nonflaky - script: make test-nonflaky
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- stage: linux_torture - stage: linux_torture
dependsOn: linux dependsOn: linux
@ -129,6 +139,8 @@ stages:
- script: make test-nonflaky - script: make test-nonflaky
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: macos_libssh2 - job: macos_libssh2
displayName: macos libssh2 displayName: macos libssh2
@ -146,6 +158,8 @@ stages:
- script: make test-nonflaky - script: make test-nonflaky
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: macos_cmake - job: macos_cmake
displayName: macos cmake openssl displayName: macos cmake openssl
@ -205,6 +219,8 @@ stages:
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys2_mingw64_debug_openssl - job: windows_msys2_mingw64_debug_openssl
displayName: msys2 mingw64 debug openssl displayName: msys2 mingw64 debug openssl
@ -226,6 +242,8 @@ stages:
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys1_mingw_debug_openssl - job: windows_msys1_mingw_debug_openssl
displayName: msys1 mingw debug openssl displayName: msys1 mingw debug openssl
@ -245,6 +263,8 @@ stages:
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys1_mingw32_debug_openssl - job: windows_msys1_mingw32_debug_openssl
displayName: msys1 mingw32 debug openssl displayName: msys1 mingw32 debug openssl
@ -264,6 +284,8 @@ stages:
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys1_mingw64_debug_openssl - job: windows_msys1_mingw64_debug_openssl
displayName: msys1 mingw64 debug openssl displayName: msys1 mingw64 debug openssl
@ -283,6 +305,8 @@ stages:
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys2_mingw32_debug_schannel - job: windows_msys2_mingw32_debug_schannel
displayName: msys2 mingw32 debug schannel displayName: msys2 mingw32 debug schannel
@ -304,6 +328,8 @@ stages:
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys2_mingw64_debug_schannel - job: windows_msys2_mingw64_debug_schannel
displayName: msys2 mingw64 debug schannel displayName: msys2 mingw64 debug schannel
@ -325,6 +351,8 @@ stages:
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys1_mingw_debug_schannel - job: windows_msys1_mingw_debug_schannel
displayName: msys1 mingw debug schannel displayName: msys1 mingw debug schannel
@ -344,6 +372,8 @@ stages:
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys1_mingw32_debug_schannel - job: windows_msys1_mingw32_debug_schannel
displayName: msys1 mingw32 debug schannel displayName: msys1 mingw32 debug schannel
@ -363,6 +393,8 @@ stages:
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
- job: windows_msys1_mingw64_debug_schannel - job: windows_msys1_mingw64_debug_schannel
displayName: msys1 mingw64 debug schannel displayName: msys1 mingw64 debug schannel
@ -382,3 +414,5 @@ stages:
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
displayName: 'test' displayName: 'test'
env:
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"

140
tests/azure.pm Normal file
View File

@ -0,0 +1,140 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
# Copyright (C) 2020, Marc Hoersken, <info@marc-hoersken.de>
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
use strict;
use warnings;
use POSIX qw(strftime);
sub azure_check_environment {
if(defined $ENV{'AZURE_ACCESS_TOKEN'} &&
defined $ENV{'AGENT_JOBNAME'} && $ENV{'BUILD_BUILDID'} &&
defined $ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'} &&
defined $ENV{'SYSTEM_TEAMPROJECTID'}) {
return 1;
}
return 0;
}
sub azure_create_test_run {
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
my $azure_run=`curl --silent \\
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
--header "Content-Type: application/json" \\
--data "
{
'name': '$ENV{'AGENT_JOBNAME'}',
'automated': true,
'build': {'id': '$ENV{'BUILD_BUILDID'}'}
}
" \\
"$azure_baseurl/_apis/test/runs?api-version=5.0"`;
if($azure_run =~ /"id":(\d+)/) {
return $1;
}
return "";
}
sub azure_create_test_result {
my ($azure_run_id, $testnum, $testname)=@_;
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
my $azure_result=`curl --silent \\
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
--header "Content-Type: application/json" \\
--data "
[
{
'build': {'id': '$ENV{'BUILD_BUILDID'}'},
'testCase': {'id': $testnum},
'testCaseTitle': '$testname',
'automatedTestName': 'curl.tests.$testnum',
'outcome': 'InProgress'
}
]
" \\
"$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`;
if($azure_result =~ /\[\{"id":(\d+)/) {
return $1;
}
return "";
}
sub azure_update_test_result {
my ($azure_run_id, $azure_result_id, $testnum, $error, $start, $stop)=@_;
if(!defined $stop) {
$stop = $start;
}
my $azure_start = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $start;
my $azure_complete = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $stop;
my $azure_duration = sprintf("%.0f", ($stop-$start)*1000);
my $azure_outcome;
if($error < 0) {
$azure_outcome = 'Not executed';
}
elsif(!$error) {
$azure_outcome = 'Passed';
}
else {
$azure_outcome = 'Failed';
}
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
my $azure_result=`curl --silent --request PATCH \\
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
--header "Content-Type: application/json" \\
--data "
[
{
'id': $azure_result_id,
'outcome': '$azure_outcome',
'startedDate': '$azure_start',
'completedDate': '$azure_complete',
'durationInMs': $azure_duration
}
]
" \\
"$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`;
if($azure_result =~ /\[\{"id":(\d+)/) {
return $1;
}
return "";
}
sub azure_update_test_run {
my ($azure_run_id)=@_;
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
my $azure_run=`curl --silent --request PATCH \\
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
--header "Content-Type: application/json" \\
--data "
{
'state': 'Completed'
}
" \\
"$azure_baseurl/_apis/test/runs/$azure_run_id?api-version=5.0"`;
if($azure_run =~ /"id":(\d+)/) {
return $1;
}
return "";
}
1;

33
tests/runtests.pl
View File

@ -111,6 +111,7 @@ use pathhelp;
require "getpart.pm"; # array functions require "getpart.pm"; # array functions
require "valgrind.pm"; # valgrind report parser require "valgrind.pm"; # valgrind report parser
require "ftp.pm"; require "ftp.pm";
require "azure.pm";
my $HOSTIP="127.0.0.1"; # address on which the test server listens my $HOSTIP="127.0.0.1"; # address on which the test server listens
my $HOST6IP="[::1]"; # address on which the test server listens my $HOST6IP="[::1]"; # address on which the test server listens
@ -327,6 +328,10 @@ my $tortalloc;
my $shallow; my $shallow;
my $randseed = 0; my $randseed = 0;
# Azure Pipelines specific variables
my $AZURE_RUN_ID = 0;
my $AZURE_RESULT_ID = 0;
####################################################################### #######################################################################
# logmsg is our general message logging subroutine. # logmsg is our general message logging subroutine.
# #
@ -3732,6 +3737,10 @@ sub singletest {
close(GDBCMD); close(GDBCMD);
} }
if(azure_check_environment() && $AZURE_RUN_ID) {
$AZURE_RESULT_ID = azure_create_test_result($AZURE_RUN_ID, $testnum, $testname);
}
# timestamp starting of test command # timestamp starting of test command
$timetoolini{$testnum} = Time::HiRes::time(); $timetoolini{$testnum} = Time::HiRes::time();
@ -4346,7 +4355,6 @@ sub singletest {
logmsg "PASS: $testnum - $testname\n"; logmsg "PASS: $testnum - $testname\n";
} }
return 0; return 0;
} }
@ -5504,6 +5512,15 @@ sub displaylogs {
} }
} }
#######################################################################
# Setup Azure Pipelines Test Run (if running in Azure DevOps)
#
if(azure_check_environment()) {
$AZURE_RUN_ID = azure_create_test_run();
logmsg "Azure Run ID: $AZURE_RUN_ID\n" if ($verbose);
}
####################################################################### #######################################################################
# The main test-loop # The main test-loop
# #
@ -5524,6 +5541,12 @@ foreach $testnum (@at) {
$count++; $count++;
my $error = singletest($run_event_based, $testnum, $count, scalar(@at)); my $error = singletest($run_event_based, $testnum, $count, scalar(@at));
if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
$AZURE_RESULT_ID = azure_update_test_result($AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
$timeprepini{$testnum}, $timevrfyend{$testnum});
}
if($error < 0) { if($error < 0) {
# not a test we can run # not a test we can run
next; next;
@ -5552,6 +5575,14 @@ foreach $testnum (@at) {
my $sofar = time() - $start; my $sofar = time() - $start;
#######################################################################
# Finish Azure Pipelines Test Run (if running in Azure DevOps)
#
if(azure_check_environment() && $AZURE_RUN_ID) {
$AZURE_RUN_ID = azure_update_test_run($AZURE_RUN_ID);
}
####################################################################### #######################################################################
# Close command log # Close command log
# #