tests: provide docs a as curldown, not nroff
As runtests.md and testcurl.md. Very few people actually need these as manpages anyway. With this, we have no more nroff formatted documents in git. Closes #14324
This commit is contained in:
Daniel Stenberg
parent
a9f63b8e0e
commit
8f89218b19
3
.github/scripts/spellcheck.words
vendored
3
.github/scripts/spellcheck.words
vendored
@ -967,3 +967,6 @@ zsh
|
|||||||
zstd
|
zstd
|
||||||
Zuul
|
Zuul
|
||||||
zuul
|
zuul
|
||||||
|
crontab
|
||||||
|
testcurl
|
||||||
|
lldb
|
||||||
|
|||||||
@ -22,7 +22,14 @@
|
|||||||
#
|
#
|
||||||
###########################################################################
|
###########################################################################
|
||||||
|
|
||||||
MANDISTPAGES = runtests.1.dist testcurl.1.dist
|
if BUILD_DOCS
|
||||||
|
# if we disable man page building, ignore these
|
||||||
|
RUNTESTS_DOCS = runtests.1
|
||||||
|
TESTCURL_DOCS = testcurl.1
|
||||||
|
MANFILES = $(RUNTESTS_DOCS) $(TESTCURL_DOCS)
|
||||||
|
endif
|
||||||
|
|
||||||
|
CURLPAGES = runtests.md testcurl.md
|
||||||
|
|
||||||
# scripts used in test cases
|
# scripts used in test cases
|
||||||
TESTSCRIPTS = \
|
TESTSCRIPTS = \
|
||||||
@ -71,7 +78,6 @@ EXTRA_DIST = \
|
|||||||
requirements.txt \
|
requirements.txt \
|
||||||
rtspserver.pl \
|
rtspserver.pl \
|
||||||
runner.pm \
|
runner.pm \
|
||||||
runtests.1 \
|
|
||||||
runtests.pl \
|
runtests.pl \
|
||||||
secureserver.pl \
|
secureserver.pl \
|
||||||
serverhelp.pm \
|
serverhelp.pm \
|
||||||
@ -80,14 +86,14 @@ EXTRA_DIST = \
|
|||||||
sshhelp.pm \
|
sshhelp.pm \
|
||||||
sshserver.pl \
|
sshserver.pl \
|
||||||
stunnel.pem \
|
stunnel.pem \
|
||||||
testcurl.1 \
|
|
||||||
testcurl.pl \
|
testcurl.pl \
|
||||||
testutil.pm \
|
testutil.pm \
|
||||||
tftpserver.pl \
|
tftpserver.pl \
|
||||||
util.py \
|
util.py \
|
||||||
valgrind.pm \
|
valgrind.pm \
|
||||||
valgrind.supp \
|
valgrind.supp \
|
||||||
$(TESTSCRIPTS)
|
$(TESTSCRIPTS) \
|
||||||
|
$(CURLPAGES)
|
||||||
|
|
||||||
DISTCLEANFILES = configurehelp.pm
|
DISTCLEANFILES = configurehelp.pm
|
||||||
|
|
||||||
@ -130,6 +136,13 @@ TEST_NF = -a -p ~flaky ~timing-dependent
|
|||||||
TEST_CI = $(TEST_NF) -r -rm
|
TEST_CI = $(TEST_NF) -r -rm
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
CD2NROFF = $(top_srcdir)/scripts/cd2nroff $< >$@
|
||||||
|
|
||||||
|
SUFFIXES = .1 .md
|
||||||
|
|
||||||
|
.md.1:
|
||||||
|
$(CD2)$(CD2NROFF)
|
||||||
|
|
||||||
# make sure that PERL is pointing to an executable
|
# make sure that PERL is pointing to an executable
|
||||||
perlcheck:
|
perlcheck:
|
||||||
@if ! test -x "$(PERL)"; then echo "No perl!"; exit 2; fi
|
@if ! test -x "$(PERL)"; then echo "No perl!"; exit 2; fi
|
||||||
@ -164,7 +177,7 @@ checksrc:
|
|||||||
(cd server && $(MAKE) checksrc)
|
(cd server && $(MAKE) checksrc)
|
||||||
(cd http && $(MAKE) checksrc)
|
(cd http && $(MAKE) checksrc)
|
||||||
|
|
||||||
if DEBUGBUILD
|
all-local: $(MANFILES)
|
||||||
# for debug builds, we scan the sources on all regular make invokes
|
|
||||||
all-local: checksrc
|
distclean:
|
||||||
endif
|
rm -f $(MANFILES)
|
||||||
|
|||||||
230
tests/runtests.1
230
tests/runtests.1
@ -1,230 +0,0 @@
|
|||||||
.\" **************************************************************************
|
|
||||||
.\" * _ _ ____ _
|
|
||||||
.\" * Project ___| | | | _ \| |
|
|
||||||
.\" * / __| | | | |_) | |
|
|
||||||
.\" * | (__| |_| | _ <| |___
|
|
||||||
.\" * \___|\___/|_| \_\_____|
|
|
||||||
.\" *
|
|
||||||
.\" * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
|
||||||
.\" *
|
|
||||||
.\" * This software is licensed as described in the file COPYING, which
|
|
||||||
.\" * you should have received as part of this distribution. The terms
|
|
||||||
.\" * are also available at https://curl.se/docs/copyright.html.
|
|
||||||
.\" *
|
|
||||||
.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
|
||||||
.\" * copies of the Software, and permit persons to whom the Software is
|
|
||||||
.\" * furnished to do so, under the terms of the COPYING file.
|
|
||||||
.\" *
|
|
||||||
.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
|
||||||
.\" * KIND, either express or implied.
|
|
||||||
.\" *
|
|
||||||
.\" * SPDX-License-Identifier: curl
|
|
||||||
.\" *
|
|
||||||
.\" **************************************************************************
|
|
||||||
.\"
|
|
||||||
.TH runtests.pl 1 "06 Jun 2023" runtests runtests
|
|
||||||
.SH NAME
|
|
||||||
runtests.pl \- run one or more test cases
|
|
||||||
.SH SYNOPSIS
|
|
||||||
.B runtests.pl [options] [tests]
|
|
||||||
.SH DESCRIPTION
|
|
||||||
\fIruntests.pl\fP runs one, several or all the existing test cases in curl's
|
|
||||||
test suite. It is often called from the root Makefile of the curl package with
|
|
||||||
\&'make test'.
|
|
||||||
.SH "TESTS"
|
|
||||||
Specify which test(s) to run by specifying test numbers or keywords.
|
|
||||||
|
|
||||||
If no test number or keyword is given, all existing tests that the script can
|
|
||||||
find will be considered for running. You can specify single test cases to run
|
|
||||||
by specifying test numbers space-separated, like "1 3 5 7 11", and you can
|
|
||||||
specify a range of tests like "45 to 67".
|
|
||||||
|
|
||||||
Specify tests to not run with a leading exclamation point, like "!66", which
|
|
||||||
runs all available tests except number 66.
|
|
||||||
|
|
||||||
Prefix a test number with a tilde (~) to still run it, but ignore the results.
|
|
||||||
|
|
||||||
It is also possible to specify tests based on a keyword describing the test(s)
|
|
||||||
to run, like "FTPS". The keywords are strings used in the individual tests.
|
|
||||||
|
|
||||||
You can also specify keywords with a leading exclamation point and the keyword
|
|
||||||
or phrase, like "!HTTP NTLM auth" to run all tests \fBexcept\fP those using
|
|
||||||
this keyword. Remember that the exclamation marks and spaces will need to be
|
|
||||||
quoted somehow when entered at many command shells.
|
|
||||||
|
|
||||||
Prefix a keyword with a tilde (~) to still run it, but ignore the results.
|
|
||||||
|
|
||||||
.SH "OUTPUT"
|
|
||||||
|
|
||||||
When running without -s (short output), for instance when running runtests.pl
|
|
||||||
directly rather than via make, each test will emits a pair of lines like this:
|
|
||||||
|
|
||||||
Test 0045...[simple HTTP Location: without protocol in initial URL]
|
|
||||||
--pd---e-v- OK (45 out of 1427, remaining: 16:08, took 6.188s, duration: 00:31)
|
|
||||||
|
|
||||||
the first line contains the test number and a description. On the second line,
|
|
||||||
the characters at the beginning are flags indicating which aspects of curl's
|
|
||||||
behavior were checked by the test:
|
|
||||||
|
|
||||||
s stdout
|
|
||||||
r stderr
|
|
||||||
p protocol
|
|
||||||
d data
|
|
||||||
u upload
|
|
||||||
P proxy
|
|
||||||
o output
|
|
||||||
e exit code
|
|
||||||
m memory
|
|
||||||
v valgrind
|
|
||||||
E the test was run event-based
|
|
||||||
|
|
||||||
The remainder of the second line contains the test result, current test sequence,
|
|
||||||
total number of tests to be run and an estimated amount of time to complete the
|
|
||||||
test run.
|
|
||||||
|
|
||||||
.SH OPTIONS
|
|
||||||
.IP "-a"
|
|
||||||
Continue running the rest of the test cases even if one test fails. By
|
|
||||||
default, the test script stops as soon as an error is detected.
|
|
||||||
.IP "-ac <curl>"
|
|
||||||
Provide a path to a curl binary to talk to APIs (currently only CI test APIs).
|
|
||||||
.IP "-am"
|
|
||||||
Display test results in automake style output (PASS/FAIL: [number] [name]).
|
|
||||||
.IP "-c <curl>"
|
|
||||||
Provide a path to a custom curl binary to run the tests with. Default is the
|
|
||||||
curl executable in the build tree.
|
|
||||||
.IP "-d"
|
|
||||||
Enable protocol debug: have the servers display protocol output. If used in
|
|
||||||
conjunction with parallel testing, it will be difficult to associate the logs
|
|
||||||
with the test being run.
|
|
||||||
.IP "-E <exclude_file>"
|
|
||||||
Load the \fBexclude_file\fP with additional reasons why certain tests
|
|
||||||
should be skipped. Useful when testing with external HTTP proxies in
|
|
||||||
which case some of the tests aren't appropriate.
|
|
||||||
The file contains colon-delimited lines. The first field contains the
|
|
||||||
type of exclusion, the second field contains a pattern and the final
|
|
||||||
field contains the reason why matching tests should be skipped.
|
|
||||||
The exclusion types are \fkeyword\fP, \ftest\fP, and \ftool\fP.
|
|
||||||
.IP "-e"
|
|
||||||
Run the test event-based (if possible). This will make runtests invoke curl
|
|
||||||
with --test-event option. This option only works if both curl and libcurl were
|
|
||||||
built debug-enabled.
|
|
||||||
.IP "-f"
|
|
||||||
Force the test to run even if mentioned in DISABLED.
|
|
||||||
.IP "-g"
|
|
||||||
Run the given test(s) with gdb. This is best used on a single test case and
|
|
||||||
curl built --disable-shared. This then fires up gdb with command line set to
|
|
||||||
run the specified test case. Simply (set a break-point and) type 'run' to
|
|
||||||
start.
|
|
||||||
.IP "-gl"
|
|
||||||
Run the given test(s) with lldb. This is best used on a single test case and
|
|
||||||
curl built --disable-shared. This then fires up lldb with command line set to
|
|
||||||
run the specified test case. Simply (set a break-point and) type 'run' to
|
|
||||||
start.
|
|
||||||
.IP "-gw"
|
|
||||||
Run the given test(s) with gdb as a windowed application.
|
|
||||||
.IP "-h, --help"
|
|
||||||
Displays a help text about this program's command line options.
|
|
||||||
.IP "-j[num]"
|
|
||||||
Spawn num processes to run tests. This defaults to 0 to run tests serially
|
|
||||||
within a single process. Using a number greater than one allows multiple tests
|
|
||||||
to run in parallel, speeding up a test run. The optimum number is dependent on
|
|
||||||
the system and set of tests to run, but 7*number of CPU cores is a good figure
|
|
||||||
to start with, or 1.3*number of CPU cores if Valgrind is in use. Enabling
|
|
||||||
parallel tests is not recommended in conjunction with the \-g option.
|
|
||||||
.IP "-k"
|
|
||||||
Keep output and log files in log/ after a test run, even if no error was
|
|
||||||
detected. Useful for debugging.
|
|
||||||
.IP "-L <file>"
|
|
||||||
Load and execute the specified file which should contain perl code.
|
|
||||||
This option allows one to change \fIruntests.pl\fP behaviour by overwriting
|
|
||||||
functions and variables and is useful when testing external proxies
|
|
||||||
using curl's regression test suite.
|
|
||||||
.IP "-l"
|
|
||||||
Lists all test case names.
|
|
||||||
.IP "-n"
|
|
||||||
Disable the check for and use of valgrind.
|
|
||||||
.IP "--no-debuginfod"
|
|
||||||
Delete the DEBUGINFOD_URLS variable if that is defined. Makes valgrind, gdb
|
|
||||||
etc not able to use this functionality.
|
|
||||||
.IP "-o <variablename=value>"
|
|
||||||
Overwrite the specified internal \fBvariable\fP with \fBvalue\fP.
|
|
||||||
Useful to change variables that didn't get a dedicated flag to change them.
|
|
||||||
Check the source to see which variables are available.
|
|
||||||
.IP "-P <proxy>"
|
|
||||||
Use the specified HTTP proxy when executing tests, even if the tests
|
|
||||||
themselves don't specify a proxy. This option allows one to test external
|
|
||||||
proxies using curl's regression test suite.
|
|
||||||
.IP "-p"
|
|
||||||
Prints out all files in "log/" to stdout when a test case fails. Very
|
|
||||||
practical when used in the automated and distributed tests since then the
|
|
||||||
people checking the failures and the reasons for them might not have physical
|
|
||||||
access to the machine and logs.
|
|
||||||
.IP "-R"
|
|
||||||
Run the tests in a scrambled, or randomized, order instead of sequentially.
|
|
||||||
|
|
||||||
The random seed initially set for this is fixed per month and can be set with
|
|
||||||
\fI--seed\fP.
|
|
||||||
.IP "-r"
|
|
||||||
Display run time statistics. (Requires Perl Time::HiRes module)
|
|
||||||
.IP "-rf"
|
|
||||||
Display full run time statistics. (Requires Perl Time::HiRes module)
|
|
||||||
.IP "-rm"
|
|
||||||
Force removal of files by killing locking processes. (Windows only,
|
|
||||||
requires Sysinternals handle[64].exe to be on PATH)
|
|
||||||
.IP "--repeat=[num]"
|
|
||||||
This will repeat the given set of test numbers this many times. If no test
|
|
||||||
numbers are given, it will repeat ALL tests this many times. It iteratively
|
|
||||||
adds the new sequence at the end of the initially given one.
|
|
||||||
|
|
||||||
If \fB-R\fP is also used, the scrambling is done after the repeats have
|
|
||||||
extended the test sequence.
|
|
||||||
.IP "-s"
|
|
||||||
Shorter output. Speaks less than default.
|
|
||||||
.IP "--seed=[num]"
|
|
||||||
When using \fI--shallow\fP or \fI-R\fP that randomize certain aspects of the
|
|
||||||
behavior, this option can set the initial seed. If not set, the random seed
|
|
||||||
will be set based on the currently set local year and month and the first line
|
|
||||||
of the "curl -V" output.
|
|
||||||
.IP "--shallow=[num]"
|
|
||||||
Used together with \fB-t\fP. This limits the number of tests to fail in
|
|
||||||
torture mode to no more than 'num' per test case. If this reduces the amount,
|
|
||||||
the script will randomly discard entries to fail until the amount is 'num'.
|
|
||||||
|
|
||||||
The random seed initially set for this is fixed per month and can be set with
|
|
||||||
\fI--seed\fP.
|
|
||||||
.IP "-t[num]"
|
|
||||||
Selects a \fBtorture\fP test for the given tests. This makes runtests.pl first
|
|
||||||
run the tests once and count the number of memory allocations made. It then
|
|
||||||
reruns the test that number of times, each time forcing one of the allocations
|
|
||||||
to fail until all allocs have been tested. By setting \fInum\fP you can force
|
|
||||||
the allocation with that number to be set to fail at once instead of looping
|
|
||||||
through everyone, which is very handy when debugging and then often in
|
|
||||||
combination with \fI-g\fP.
|
|
||||||
.IP "-u"
|
|
||||||
Error instead of warning on server unexpectedly alive.
|
|
||||||
.IP "-v"
|
|
||||||
Enable verbose output. Speaks more than by default. If used in conjunction with
|
|
||||||
parallel testing, it will be difficult to associate the logs with the test
|
|
||||||
being run.
|
|
||||||
.IP "-vc <curl>"
|
|
||||||
Provide a path to a custom curl binary to run when verifying that the servers
|
|
||||||
running are indeed our test servers. Default is the curl executable in the
|
|
||||||
build tree.
|
|
||||||
.SH "RUNNING TESTS"
|
|
||||||
Many tests have conditions that must be met before the test case can run
|
|
||||||
fine. They could depend on built-in features in libcurl or features present in
|
|
||||||
the operating system or even in third-party libraries that curl may or may not
|
|
||||||
use.
|
|
||||||
.P
|
|
||||||
The test script checks most of these by itself to determine when it is
|
|
||||||
safe to attempt to run each test. Those which cannot be run due to
|
|
||||||
failed requirements will simply be skipped and listed at the completion
|
|
||||||
of all test cases. In some unusual configurations, the test script
|
|
||||||
cannot make the correct determination for all tests. In these cases,
|
|
||||||
the problematic tests can be skipped using the "!keyword" skip feature
|
|
||||||
documented earlier.
|
|
||||||
.SH "WRITING TESTS"
|
|
||||||
The simplest way to write test cases is to start with a similar existing test,
|
|
||||||
save it with a new number and then adjust it to fit. There's an attempt to
|
|
||||||
document the test case file format in the tests/FILEFORMAT.md.
|
|
||||||
295
tests/runtests.md
Normal file
295
tests/runtests.md
Normal file
@ -0,0 +1,295 @@
|
|||||||
|
---
|
||||||
|
c: Copyright (C) Daniel Stenberg, <daniel.se>, et al.
|
||||||
|
SPDX-License-Identifier: curl
|
||||||
|
Title: runtests.pl
|
||||||
|
Section: 1
|
||||||
|
Source: runtests
|
||||||
|
See-also:
|
||||||
|
- runtests.pl
|
||||||
|
Added-in: 7.5
|
||||||
|
---
|
||||||
|
|
||||||
|
# NAME
|
||||||
|
|
||||||
|
runtests.pl - run one or more test cases
|
||||||
|
|
||||||
|
# SYNOPSIS
|
||||||
|
|
||||||
|
**runtests.pl [options] [tests]**
|
||||||
|
|
||||||
|
# DESCRIPTION
|
||||||
|
|
||||||
|
*runtests.pl* runs one, several or all the existing test cases in curl's
|
||||||
|
test suite. It is often called from the root Makefile of the curl package with
|
||||||
|
'make test'.
|
||||||
|
|
||||||
|
# TESTS
|
||||||
|
|
||||||
|
Specify which test(s) to run by specifying test numbers or keywords.
|
||||||
|
|
||||||
|
If no test number or keyword is given, all existing tests that the script can
|
||||||
|
find will be considered for running. You can specify single test cases to run
|
||||||
|
by specifying test numbers space-separated, like `1 3 5 7 11`, and you can
|
||||||
|
specify a range of tests like `45 to 67`.
|
||||||
|
|
||||||
|
Specify tests to not run with a leading exclamation point, like `!66`, which
|
||||||
|
runs all available tests except number 66.
|
||||||
|
|
||||||
|
Prefix a test number with a tilde (~) to still run it, but ignore the results.
|
||||||
|
|
||||||
|
It is also possible to specify tests based on a keyword describing the test(s)
|
||||||
|
to run, like `FTPS`. The keywords are strings used in the individual tests.
|
||||||
|
|
||||||
|
You can also specify keywords with a leading exclamation point and the keyword
|
||||||
|
or phrase, like "!HTTP NTLM auth" to run all tests **except** those using
|
||||||
|
this keyword. Remember that the exclamation marks and spaces will need to be
|
||||||
|
quoted somehow when entered at many command shells.
|
||||||
|
|
||||||
|
Prefix a keyword with a tilde (~) to still run it, but ignore the results.
|
||||||
|
|
||||||
|
# OUTPUT
|
||||||
|
|
||||||
|
When running without `-s` (short output), for instance when running
|
||||||
|
runtests.pl directly rather than via make, each test will emits a pair of
|
||||||
|
lines like this:
|
||||||
|
|
||||||
|
Test 0045...[simple HTTP Location: without protocol in initial URL]
|
||||||
|
--pd---e-v- OK (45 out of 1427, remaining: 16:08, took 6.188s, duration: 00:31)
|
||||||
|
|
||||||
|
the first line contains the test number and a description. On the second line,
|
||||||
|
the characters at the beginning are flags indicating which aspects of curl's
|
||||||
|
behavior were checked by the test:
|
||||||
|
|
||||||
|
s stdout
|
||||||
|
r stderr
|
||||||
|
p protocol
|
||||||
|
d data
|
||||||
|
u upload
|
||||||
|
P proxy
|
||||||
|
o output
|
||||||
|
e exit code
|
||||||
|
m memory
|
||||||
|
v valgrind
|
||||||
|
E the test was run event-based
|
||||||
|
|
||||||
|
The remainder of the second line contains the test result, current test sequence,
|
||||||
|
total number of tests to be run and an estimated amount of time to complete the
|
||||||
|
test run.
|
||||||
|
|
||||||
|
# OPTIONS
|
||||||
|
|
||||||
|
## `-a`
|
||||||
|
|
||||||
|
Continue running the rest of the test cases even if one test fails. By
|
||||||
|
default, the test script stops as soon as an error is detected.
|
||||||
|
|
||||||
|
## `-ac \<curl\>`
|
||||||
|
|
||||||
|
Provide a path to a curl binary to talk to APIs (currently only CI test APIs).
|
||||||
|
|
||||||
|
## `-am`
|
||||||
|
|
||||||
|
Display test results in automake style output (`PASS/FAIL: [number] [name]`).
|
||||||
|
|
||||||
|
## `-c\<curl\>`
|
||||||
|
|
||||||
|
Provide a path to a custom curl binary to run the tests with. Default is the
|
||||||
|
curl executable in the build tree.
|
||||||
|
|
||||||
|
## `-d`
|
||||||
|
|
||||||
|
Enable protocol debug: have the servers display protocol output. If used in
|
||||||
|
conjunction with parallel testing, it will be difficult to associate the logs
|
||||||
|
with the test being run.
|
||||||
|
|
||||||
|
## `-E \<exclude_file\>`
|
||||||
|
|
||||||
|
Load the **exclude_file** with additional reasons why certain tests should be
|
||||||
|
skipped. Useful when testing with external HTTP proxies in which case some of
|
||||||
|
the tests aren't appropriate.
|
||||||
|
|
||||||
|
The file contains colon-delimited lines. The first field contains the type of
|
||||||
|
exclusion, the second field contains a pattern and the final field contains
|
||||||
|
the reason why matching tests should be skipped. The exclusion types are
|
||||||
|
*keyword*, *test*, and *tool*.
|
||||||
|
|
||||||
|
## `-e`
|
||||||
|
|
||||||
|
Run the test event-based (if possible). This will make runtests invoke curl
|
||||||
|
with --test-event option. This option only works if both curl and libcurl were
|
||||||
|
built debug-enabled.
|
||||||
|
|
||||||
|
## `-f`
|
||||||
|
|
||||||
|
Force the test to run even if mentioned in DISABLED.
|
||||||
|
|
||||||
|
## `-g`
|
||||||
|
|
||||||
|
Run the given test(s) with gdb. This is best used on a single test case and
|
||||||
|
curl built --disable-shared. This then fires up gdb with command line set to
|
||||||
|
run the specified test case. Simply (set a break-point and) type 'run' to
|
||||||
|
start.
|
||||||
|
|
||||||
|
## `-gl`
|
||||||
|
|
||||||
|
Run the given test(s) with lldb. This is best used on a single test case and
|
||||||
|
curl built --disable-shared. This then fires up lldb with command line set to
|
||||||
|
run the specified test case. Simply (set a break-point and) type 'run' to
|
||||||
|
start.
|
||||||
|
|
||||||
|
## `-gw`
|
||||||
|
|
||||||
|
Run the given test(s) with gdb as a windowed application.
|
||||||
|
|
||||||
|
## `-h, --help`
|
||||||
|
|
||||||
|
Displays a help text about this program's command line options.
|
||||||
|
|
||||||
|
## `-j[num]`
|
||||||
|
|
||||||
|
Spawn the given number of processes to run tests in. This defaults to 0 to run
|
||||||
|
tests serially within a single process. Using a number greater than one allows
|
||||||
|
multiple tests to run in parallel, speeding up a test run. The optimum number
|
||||||
|
is dependent on the system and set of tests to run, but 7 times number of CPU
|
||||||
|
cores is a good figure to start with, or 1.3 times number of CPU cores if
|
||||||
|
Valgrind is in use. Enabling parallel tests is not recommended in conjunction
|
||||||
|
with the -g option.
|
||||||
|
|
||||||
|
## `-k`
|
||||||
|
|
||||||
|
Keep output and log files in log/ after a test run, even if no error was
|
||||||
|
detected. Useful for debugging.
|
||||||
|
|
||||||
|
## `-L \<file\>`
|
||||||
|
|
||||||
|
Load and execute the specified file which should contain perl code. This
|
||||||
|
option allows one to change *runtests.pl* behavior by overwriting functions
|
||||||
|
and variables and is useful when testing external proxies using curl's
|
||||||
|
regression test suite.
|
||||||
|
|
||||||
|
## `-l`
|
||||||
|
|
||||||
|
Lists all test case names.
|
||||||
|
|
||||||
|
## `-n`
|
||||||
|
|
||||||
|
Disable the check for and use of valgrind.
|
||||||
|
|
||||||
|
## `--no-debuginfod`
|
||||||
|
|
||||||
|
Delete the `DEBUGINFOD_URLS` variable if that is defined. Makes valgrind, gdb
|
||||||
|
etc not able to use this functionality.
|
||||||
|
|
||||||
|
## `-o \<variablename=value\>`
|
||||||
|
|
||||||
|
Overwrite the specified internal **variable** with **value**. Useful to change
|
||||||
|
variables that didn't get a dedicated flag to change them. Check the source to
|
||||||
|
see which variables are available.
|
||||||
|
|
||||||
|
## `-P \<proxy\>`
|
||||||
|
|
||||||
|
Use the specified HTTP proxy when executing tests, even if the tests
|
||||||
|
themselves don't specify a proxy. This option allows one to test external
|
||||||
|
proxies using curl's regression test suite.
|
||||||
|
|
||||||
|
## `-p`
|
||||||
|
|
||||||
|
Prints out all files in the log directory to stdout when a test case fails.
|
||||||
|
Practical when used in the automated and distributed tests since then the
|
||||||
|
people checking the failures and the reasons for them might not have physical
|
||||||
|
access to the machine and logs.
|
||||||
|
|
||||||
|
## `-R`
|
||||||
|
|
||||||
|
Run the tests in a scrambled, or randomized, order instead of sequentially.
|
||||||
|
|
||||||
|
The random seed initially set for this is fixed per month and can be set with
|
||||||
|
*--seed*.
|
||||||
|
|
||||||
|
## `-r`
|
||||||
|
|
||||||
|
Display run time statistics. (Requires the `Perl Time::HiRes` module)
|
||||||
|
|
||||||
|
## `-rf`
|
||||||
|
|
||||||
|
Display full run time statistics. (Requires the `Perl Time::HiRes` module)
|
||||||
|
|
||||||
|
## `-rm`
|
||||||
|
|
||||||
|
Force removal of files by killing locking processes. (Windows only, requires
|
||||||
|
the **Sysinternals** `handle[64].exe` to be on PATH)
|
||||||
|
|
||||||
|
## `--repeat=[num]`
|
||||||
|
|
||||||
|
This repeats the given set of test numbers this many times. If no test numbers
|
||||||
|
are given, it will repeat ALL tests this many times. It adds the new repeated
|
||||||
|
sequence at the end of the initially given one.
|
||||||
|
|
||||||
|
If **-R** option is also used, the scrambling is done after the repeats have
|
||||||
|
extended the test sequence.
|
||||||
|
|
||||||
|
## `-s`
|
||||||
|
|
||||||
|
Shorter output. Speaks less than default.
|
||||||
|
|
||||||
|
## `--seed=[num]`
|
||||||
|
|
||||||
|
When using *--shallow* or *-R* that randomize certain aspects of the
|
||||||
|
behavior, this option can set the initial seed. If not set, the random seed
|
||||||
|
will be set based on the currently set local year and month and the first line
|
||||||
|
of the "curl -V" output.
|
||||||
|
|
||||||
|
## `--shallow=[num]`
|
||||||
|
|
||||||
|
Used together with **-t**. This limits the number of tests to fail in torture
|
||||||
|
mode to no more than **num** per test case. If this reduces the amount, the
|
||||||
|
script will randomly discard entries to fail until the amount is **num**.
|
||||||
|
|
||||||
|
The random seed initially set for this is fixed per month and can be set with
|
||||||
|
*--seed*.
|
||||||
|
|
||||||
|
## `-t[num]`
|
||||||
|
|
||||||
|
Selects a **torture** test for the given tests. This makes runtests.pl first
|
||||||
|
run the tests once and count the number of memory allocations made. It then
|
||||||
|
reruns the test that number of times, each time forcing one of the allocations
|
||||||
|
to fail until all allocations have been tested. By setting *num* you can force
|
||||||
|
the allocation with that number to be set to fail at once instead of looping
|
||||||
|
through everyone, which is handy when debugging and then often in combination
|
||||||
|
with *-g*.
|
||||||
|
|
||||||
|
## `-u`
|
||||||
|
|
||||||
|
Error instead of warning on server unexpectedly alive.
|
||||||
|
|
||||||
|
## `-v`
|
||||||
|
|
||||||
|
Enable verbose output. Speaks more than by default. If used in conjunction with
|
||||||
|
parallel testing, it will be difficult to associate the logs with the test
|
||||||
|
being run.
|
||||||
|
|
||||||
|
## `-vc \<curl\>`
|
||||||
|
|
||||||
|
Provide a path to a custom curl binary to run when verifying that the servers
|
||||||
|
running are indeed our test servers. Default is the curl executable in the
|
||||||
|
build tree.
|
||||||
|
|
||||||
|
# RUNNING TESTS
|
||||||
|
|
||||||
|
Many tests have conditions that must be met before the test case can run fine.
|
||||||
|
They could depend on built-in features in libcurl or features present in the
|
||||||
|
operating system or even in third-party libraries that curl may or may not
|
||||||
|
use.
|
||||||
|
|
||||||
|
The test script checks most of these by itself to determine when it is safe to
|
||||||
|
attempt to run each test. Those which cannot be run due to failed requirements
|
||||||
|
will simply be skipped and listed at the completion of all test cases. In some
|
||||||
|
unusual configurations, the test script cannot make the correct determination
|
||||||
|
for all tests. In these cases, the problematic tests can be skipped using the
|
||||||
|
"!keyword" skip feature documented earlier.
|
||||||
|
|
||||||
|
# WRITING TESTS
|
||||||
|
|
||||||
|
The simplest way to write test cases is to start with a similar existing test,
|
||||||
|
save it with a new number and then adjust it to fit. There's an attempt to
|
||||||
|
document the test case file format in **tests/FILEFORMAT.md**.
|
||||||
126
tests/testcurl.1
126
tests/testcurl.1
@ -1,126 +0,0 @@
|
|||||||
.\" **************************************************************************
|
|
||||||
.\" * _ _ ____ _
|
|
||||||
.\" * Project ___| | | | _ \| |
|
|
||||||
.\" * / __| | | | |_) | |
|
|
||||||
.\" * | (__| |_| | _ <| |___
|
|
||||||
.\" * \___|\___/|_| \_\_____|
|
|
||||||
.\" *
|
|
||||||
.\" * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
|
||||||
.\" *
|
|
||||||
.\" * This software is licensed as described in the file COPYING, which
|
|
||||||
.\" * you should have received as part of this distribution. The terms
|
|
||||||
.\" * are also available at https://curl.se/docs/copyright.html.
|
|
||||||
.\" *
|
|
||||||
.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
|
||||||
.\" * copies of the Software, and permit persons to whom the Software is
|
|
||||||
.\" * furnished to do so, under the terms of the COPYING file.
|
|
||||||
.\" *
|
|
||||||
.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
|
||||||
.\" * KIND, either express or implied.
|
|
||||||
.\" *
|
|
||||||
.\" * SPDX-License-Identifier: curl
|
|
||||||
.\" *
|
|
||||||
.\" **************************************************************************
|
|
||||||
.\"
|
|
||||||
.TH testcurl.pl 1 "24 Mar 2010" testcurl testcurl
|
|
||||||
.SH NAME
|
|
||||||
testcurl.pl \- (automatically) test curl
|
|
||||||
.SH SYNOPSIS
|
|
||||||
.B testcurl.pl [options] [dir] > output
|
|
||||||
.SH DESCRIPTION
|
|
||||||
\fItestcurl.pl\fP is the master script to use for automatic testing of curl
|
|
||||||
off git or daily snapshots. It is written for the purpose of being run from a
|
|
||||||
crontab job or similar at a regular interval. The output is suitable to be
|
|
||||||
mailed to curl-autocompile@haxx.se to be dealt with automatically (make sure
|
|
||||||
the subject includes the word "autobuild" as the mail gets silently discarded
|
|
||||||
otherwise). The most current build status (with a reasonable backlog) will be
|
|
||||||
published on the curl site, at https://curl.se/dev/builds.html
|
|
||||||
|
|
||||||
\fIoptions\fP may be omitted. See \fI--setup\fP for what happens then.
|
|
||||||
|
|
||||||
\fIdir\fP is a curl source dir, possibly a daily snapshot one. Using this will
|
|
||||||
make testcurl.pl skip the 'buildconf' stage and thus it removes the dependency
|
|
||||||
on automake, autoconf, libtool, GNU m4 and possibly a few other things.
|
|
||||||
|
|
||||||
testcurl.pl will run 'buildconf' (or similar), run configure, build curl and
|
|
||||||
libcurl in a separate build directory and then run 'make test' to test the
|
|
||||||
fresh build.
|
|
||||||
.SH OPTIONS
|
|
||||||
.IP "--configure=[options]"
|
|
||||||
Configure options passed to configure.
|
|
||||||
.IP "--crosscompile"
|
|
||||||
This is a cross-compile. Makes \fItestcurl.pl\fP skip a few things.
|
|
||||||
.IP "--desc=[desc]"
|
|
||||||
Description of your test system. Displayed on the build summary page on the
|
|
||||||
weba site.
|
|
||||||
.IP "--email=[email]"
|
|
||||||
Set email address to report as. Displayed in the build logs on the site.
|
|
||||||
.IP "--mktarball=[command]"
|
|
||||||
Generic command to run after completed test.
|
|
||||||
.IP "--name=[name]"
|
|
||||||
Set name to report as. Displayed in the build summary on the site.
|
|
||||||
.IP "--nobuildconf"
|
|
||||||
Don't run buildconf. Useful when many builds use the same source tree, as then
|
|
||||||
only one need to do this. Also, if multiple processes run tests simultaneously
|
|
||||||
on the same source tree (like several hosts on a NFS mounted dir),
|
|
||||||
simultaneous buildconf invokes may cause problems. (Added in 7.14.1)
|
|
||||||
.IP "--nogitpull"
|
|
||||||
Don't update from git even though it is a git tree. Useful to still be able to
|
|
||||||
test even though your network is down, or similar.
|
|
||||||
.IP "--runtestopts=[options]"
|
|
||||||
Options that is passed to the runtests.pl script. Useful for disabling valgrind
|
|
||||||
by force, and similar.
|
|
||||||
.IP "--setup=[file name]"
|
|
||||||
File name to read setup from (deprecated). The old style of providing info.
|
|
||||||
If info is missing when testcurl.pl is started, it will prompt you and then
|
|
||||||
store the info in a 'setup' file, which it will look for on each invoke. Use
|
|
||||||
\fI--name\fP, \fI--email\fP, \fI--configure\fP and \fI--desc\fP instead.
|
|
||||||
.IP "--target=[your os]"
|
|
||||||
Specify your target environment. Recognized strings include 'vc', 'mingw32',
|
|
||||||
and \&'borland'.
|
|
||||||
.SH "INITIAL SETUP"
|
|
||||||
First you make a checkout from git (or you write a script that downloads daily
|
|
||||||
snapshots automatically, find inspiration in
|
|
||||||
https://curl.se/dev/autocurl.txt ):
|
|
||||||
|
|
||||||
.nf
|
|
||||||
$ mkdir daily-curl
|
|
||||||
$ cd daily-curl
|
|
||||||
$ git clone https://github.com/curl/curl.git
|
|
||||||
.fi
|
|
||||||
|
|
||||||
With the curl sources checked out, or downloaded, you can start testing right
|
|
||||||
away. If you want to use \fItestcurl.pl\fP without command line arguments and
|
|
||||||
to have it store and remember the config in its 'setup' file, then start it
|
|
||||||
manually now and fill in the answers to the questions it prompts you for:
|
|
||||||
|
|
||||||
.nf
|
|
||||||
$ ./curl/tests/testcurl.pl
|
|
||||||
.fi
|
|
||||||
|
|
||||||
Now you are ready to go. If you let the script run, it will perform a full
|
|
||||||
cycle and spit out lots of output. Mail us that output as described above.
|
|
||||||
.SH "CRONTAB EXAMPLE"
|
|
||||||
The crontab could include something like this:
|
|
||||||
|
|
||||||
.nf
|
|
||||||
\# autobuild curl:
|
|
||||||
0 4 * * * cd daily-curl && ./testit.sh
|
|
||||||
.fi
|
|
||||||
|
|
||||||
Where testit.sh is a shell script that could look similar to this:
|
|
||||||
|
|
||||||
.nf
|
|
||||||
mail="mail -s autobuild curl-autocompile@haxx.se"
|
|
||||||
name="--name=whoami"
|
|
||||||
email="--email=iamme@nowhere"
|
|
||||||
desc='"--desc=supermachine Turbo 2000"'
|
|
||||||
testprog="perl ./curl/tests/testcurl.pl $name $email $desc"
|
|
||||||
opts1="--configure=--enable-debug"
|
|
||||||
opts2="--configure=--enable-ipv6"
|
|
||||||
|
|
||||||
# run first test
|
|
||||||
$testprog $opts1 | $mail
|
|
||||||
|
|
||||||
# run second test
|
|
||||||
$testprog $opts2 | $mail
|
|
||||||
137
tests/testcurl.md
Normal file
137
tests/testcurl.md
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
---
|
||||||
|
c: Copyright (C) Daniel Stenberg, <daniel.se>, et al.
|
||||||
|
SPDX-License-Identifier: curl
|
||||||
|
Title: testcurl.pl
|
||||||
|
Section: 1
|
||||||
|
Source: testcurl
|
||||||
|
See-also:
|
||||||
|
- runtests.pl
|
||||||
|
Added-in: 7.11.2
|
||||||
|
---
|
||||||
|
|
||||||
|
# NAME
|
||||||
|
|
||||||
|
testcurl.pl - (automatically) test curl
|
||||||
|
|
||||||
|
# SYNOPSIS
|
||||||
|
|
||||||
|
**testcurl.pl [options] [dir] \> output**
|
||||||
|
|
||||||
|
# DESCRIPTION
|
||||||
|
|
||||||
|
*testcurl* is the master script to use for automatic distributed testing of
|
||||||
|
curl from git or daily snapshots. It is written for the purpose of being run
|
||||||
|
from a crontab job or similar at a regular interval. The output is suitable to
|
||||||
|
be mailed to **curl-autocompile@haxx.se** to be dealt with automatically (make
|
||||||
|
sure the subject includes the word "autobuild" as the mail gets silently
|
||||||
|
discarded otherwise). The most current build status (with a reasonable
|
||||||
|
backlog) will be published on the curl site, at
|
||||||
|
https://curl.se/dev/builds.html
|
||||||
|
|
||||||
|
*options* may be omitted. See *--setup* for what happens then.
|
||||||
|
|
||||||
|
*dir* is a curl source dir, possibly a daily snapshot one. Using this will
|
||||||
|
make *testcurl* skip the 'buildconf' stage and thus it removes the dependency
|
||||||
|
on automake, autoconf, libtool, GNU m4 and possibly a few other things.
|
||||||
|
|
||||||
|
*testcurl* runs `buildconf` (or similar), configure, builds curl and libcurl
|
||||||
|
in a separate build directory and then runs `make test` to test the fresh
|
||||||
|
build.
|
||||||
|
|
||||||
|
# OPTIONS
|
||||||
|
|
||||||
|
## `--configure=[options]`
|
||||||
|
|
||||||
|
Configure options passed to configure.
|
||||||
|
|
||||||
|
## `--crosscompile`
|
||||||
|
``
|
||||||
|
This is a cross-compile. Makes *testcurl* skip a few things.
|
||||||
|
|
||||||
|
## `--desc=[desc]`
|
||||||
|
|
||||||
|
Description of your test system. Displayed on the build summary page on the
|
||||||
|
website.
|
||||||
|
|
||||||
|
## `--email=[email]`
|
||||||
|
|
||||||
|
Set email address to report as. Displayed in the build logs on the site.
|
||||||
|
|
||||||
|
## `--mktarball=[command]`
|
||||||
|
|
||||||
|
Generic command to run after completed test.
|
||||||
|
|
||||||
|
## `--name=[name]`
|
||||||
|
|
||||||
|
Set name to report as. Displayed in the build summary on the site.
|
||||||
|
|
||||||
|
## `--nobuildconf`
|
||||||
|
|
||||||
|
Don't run buildconf. Useful when many builds use the same source tree, as then
|
||||||
|
only one need to do this. Also, if multiple processes run tests simultaneously
|
||||||
|
on the same source tree (like several hosts on a NFS mounted dir),
|
||||||
|
simultaneous buildconf invokes may cause problems. (Added in 7.14.1)
|
||||||
|
|
||||||
|
## `--nogitpull`
|
||||||
|
|
||||||
|
Don't update from git even though it is a git tree. Useful to still be able to
|
||||||
|
test even though your network is down, or similar.
|
||||||
|
|
||||||
|
## `--runtestopts=[options]`
|
||||||
|
|
||||||
|
Options that is passed to the runtests script. Useful for disabling valgrind
|
||||||
|
by force, and similar.
|
||||||
|
|
||||||
|
## `--setup=[file name]`
|
||||||
|
|
||||||
|
File name to read setup from (deprecated). The old style of providing info.
|
||||||
|
If info is missing when *testcurl* is started, it will prompt you and then
|
||||||
|
store the info in a 'setup' file, which it will look for on each invoke. Use
|
||||||
|
*--name*, *--email*, *--configure* and *--desc* instead.
|
||||||
|
|
||||||
|
## `--target=[your os]`
|
||||||
|
|
||||||
|
Specify your target environment. Recognized strings include `vc`, `mingw32`,
|
||||||
|
and `borland`.
|
||||||
|
|
||||||
|
# INITIAL SETUP
|
||||||
|
|
||||||
|
First, make a checkout from git (or you write a script that downloads daily
|
||||||
|
snapshots automatically):
|
||||||
|
|
||||||
|
$ mkdir curl-testing
|
||||||
|
$ cd curl-testing
|
||||||
|
$ git clone https://github.com/curl/curl.git
|
||||||
|
|
||||||
|
With the curl sources checked out, or downloaded, you can start testing right
|
||||||
|
away. If you want to use *testcurl* without command line arguments and to have
|
||||||
|
it store and remember the config in its 'setup' file, then start it manually
|
||||||
|
now and fill in the answers to the questions it prompts you for:
|
||||||
|
|
||||||
|
$ ./curl/tests/testcurl
|
||||||
|
|
||||||
|
Now you are ready to go. If you let the script run, it will perform a full
|
||||||
|
cycle and spit out lots of output. Mail us that output as described above.
|
||||||
|
|
||||||
|
# CRONTAB EXAMPLE
|
||||||
|
|
||||||
|
The crontab could include something like this:
|
||||||
|
|
||||||
|
# autobuild curl:
|
||||||
|
0 4 * * * cd curl-testing && ./testit.sh
|
||||||
|
|
||||||
|
Where `testit.sh` is a shell script that could look similar to this:
|
||||||
|
|
||||||
|
mail="mail -s autobuild curl-autocompile@haxx.se"
|
||||||
|
name="--name=whoami"
|
||||||
|
email="--email=iamme@nowhere"
|
||||||
|
desc='"--desc=supermachine Turbo 2000"'
|
||||||
|
testprog="perl ./curl/tests/testcurl.pl $name $email $desc"
|
||||||
|
opts1="--configure=--enable-debug"
|
||||||
|
opts2="--configure=--enable-ipv6"
|
||||||
|
|
||||||
|
# run first test
|
||||||
|
$testprog $opts1 | $mail
|
||||||
|
|
||||||
|
# run second test
|
||||||
|
$testprog $opts2 | $mail
|
||||||
Loading…
Reference in New Issue
Block a user