988 lines
35 KiB
Python
988 lines
35 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
# ***************************************************************************
|
|
# _ _ ____ _
|
|
# Project ___| | | | _ \| |
|
|
# / __| | | | |_) | |
|
|
# | (__| |_| | _ <| |___
|
|
# \___|\___/|_| \_\_____|
|
|
#
|
|
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
|
#
|
|
# This software is licensed as described in the file COPYING, which
|
|
# you should have received as part of this distribution. The terms
|
|
# are also available at https://curl.se/docs/copyright.html.
|
|
#
|
|
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
|
# copies of the Software, and permit persons to whom the Software is
|
|
# furnished to do so, under the terms of the COPYING file.
|
|
#
|
|
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
|
# KIND, either express or implied.
|
|
#
|
|
# SPDX-License-Identifier: curl
|
|
#
|
|
###########################################################################
|
|
#
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
from statistics import mean
|
|
from typing import Dict, Any, Optional, List
|
|
|
|
from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class ScoreCardError(Exception):
|
|
pass
|
|
|
|
|
|
class ScoreCard:
|
|
def __init__(
|
|
self,
|
|
env: Env,
|
|
protocol: str,
|
|
server_descr: str,
|
|
server_port: int,
|
|
verbose: int,
|
|
curl_verbose: int,
|
|
download_parallel: int = 0,
|
|
server_addr: Optional[str] = None,
|
|
):
|
|
self.verbose = verbose
|
|
self.env = env
|
|
self.protocol = protocol
|
|
self.server_descr = server_descr
|
|
self.server_addr = server_addr
|
|
self.server_port = server_port
|
|
self._silent_curl = not curl_verbose
|
|
self._download_parallel = download_parallel
|
|
|
|
def info(self, msg):
|
|
if self.verbose > 0:
|
|
sys.stderr.write(msg)
|
|
sys.stderr.flush()
|
|
|
|
def handshakes(self) -> Dict[str, Any]:
|
|
props = {}
|
|
sample_size = 5
|
|
self.info("TLS Handshake\n")
|
|
for authority in ["curl.se", "google.com", "cloudflare.com", "nghttp2.org"]:
|
|
self.info(f" {authority}...")
|
|
props[authority] = {}
|
|
for ipv in ["ipv4", "ipv6"]:
|
|
self.info(f"{ipv}...")
|
|
c_samples = []
|
|
hs_samples = []
|
|
errors = []
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env,
|
|
silent=self._silent_curl,
|
|
server_addr=self.server_addr,
|
|
)
|
|
args = [
|
|
"--http3-only" if self.protocol == "h3" else "--http2",
|
|
f"--{ipv}",
|
|
f"https://{authority}/",
|
|
]
|
|
r = curl.run_direct(args=args, with_stats=True)
|
|
if r.exit_code == 0 and len(r.stats) == 1:
|
|
c_samples.append(r.stats[0]["time_connect"])
|
|
hs_samples.append(r.stats[0]["time_appconnect"])
|
|
else:
|
|
errors.append(f"exit={r.exit_code}")
|
|
props[authority][f"{ipv}-connect"] = (
|
|
mean(c_samples) if len(c_samples) else -1
|
|
)
|
|
props[authority][f"{ipv}-handshake"] = (
|
|
mean(hs_samples) if len(hs_samples) else -1
|
|
)
|
|
props[authority][f"{ipv}-errors"] = errors
|
|
self.info("ok.\n")
|
|
return props
|
|
|
|
def _make_docs_file(self, docs_dir: str, fname: str, fsize: int):
|
|
fpath = os.path.join(docs_dir, fname)
|
|
data1k = 1024 * "x"
|
|
flen = 0
|
|
with open(fpath, "w") as fd:
|
|
while flen < fsize:
|
|
fd.write(data1k)
|
|
flen += len(data1k)
|
|
return fpath
|
|
|
|
def setup_resources(self, server_docs: str, downloads: Optional[List[int]] = None):
|
|
for fsize in downloads:
|
|
label = self.fmt_size(fsize)
|
|
fname = f"score{label}.data"
|
|
self._make_docs_file(docs_dir=server_docs, fname=fname, fsize=fsize)
|
|
self._make_docs_file(docs_dir=server_docs, fname="reqs10.data", fsize=10 * 1024)
|
|
|
|
def _check_downloads(self, r: ExecResult, count: int):
|
|
error = ""
|
|
if r.exit_code != 0:
|
|
error += f"exit={r.exit_code} "
|
|
if r.exit_code != 0 or len(r.stats) != count:
|
|
error += f"stats={len(r.stats)}/{count} "
|
|
fails = [s for s in r.stats if s["response_code"] != 200]
|
|
if len(fails) > 0:
|
|
error += f"{len(fails)} failed"
|
|
return error if len(error) > 0 else None
|
|
|
|
def transfer_single(self, url: str, count: int):
|
|
sample_size = count
|
|
count = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
self.info("single...")
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env, silent=self._silent_curl, server_addr=self.server_addr
|
|
)
|
|
r = curl.http_download(
|
|
urls=[url],
|
|
alpn_proto=self.protocol,
|
|
no_save=True,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
)
|
|
err = self._check_downloads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s["size_download"] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return {
|
|
"count": count,
|
|
"samples": sample_size,
|
|
"max-parallel": 1,
|
|
"speed": mean(samples) if len(samples) else -1,
|
|
"errors": errors,
|
|
"stats": RunProfile.AverageStats(profiles),
|
|
}
|
|
|
|
def transfer_serial(self, url: str, count: int):
|
|
sample_size = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
url = f"{url}?[0-{count - 1}]"
|
|
self.info("serial...")
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env, silent=self._silent_curl, server_addr=self.server_addr
|
|
)
|
|
r = curl.http_download(
|
|
urls=[url],
|
|
alpn_proto=self.protocol,
|
|
no_save=True,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
)
|
|
err = self._check_downloads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s["size_download"] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return {
|
|
"count": count,
|
|
"samples": sample_size,
|
|
"max-parallel": 1,
|
|
"speed": mean(samples) if len(samples) else -1,
|
|
"errors": errors,
|
|
"stats": RunProfile.AverageStats(profiles),
|
|
}
|
|
|
|
def transfer_parallel(self, url: str, count: int):
|
|
sample_size = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
max_parallel = self._download_parallel if self._download_parallel > 0 else count
|
|
url = f"{url}?[0-{count - 1}]"
|
|
self.info("parallel...")
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env, silent=self._silent_curl, server_addr=self.server_addr
|
|
)
|
|
r = curl.http_download(
|
|
urls=[url],
|
|
alpn_proto=self.protocol,
|
|
no_save=True,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
extra_args=["--parallel", "--parallel-max", str(max_parallel)],
|
|
)
|
|
err = self._check_downloads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s["size_download"] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return {
|
|
"count": count,
|
|
"samples": sample_size,
|
|
"max-parallel": max_parallel,
|
|
"speed": mean(samples) if len(samples) else -1,
|
|
"errors": errors,
|
|
"stats": RunProfile.AverageStats(profiles),
|
|
}
|
|
|
|
def download_url(self, label: str, url: str, count: int):
|
|
self.info(f" {count}x{label}: ")
|
|
props = {
|
|
"single": self.transfer_single(url=url, count=10),
|
|
}
|
|
if count > 1:
|
|
props["serial"] = self.transfer_serial(url=url, count=count)
|
|
props["parallel"] = self.transfer_parallel(url=url, count=count)
|
|
self.info("ok.\n")
|
|
return props
|
|
|
|
def downloads(self, count: int, fsizes: List[int]) -> Dict[str, Any]:
|
|
scores = {}
|
|
for fsize in fsizes:
|
|
label = self.fmt_size(fsize)
|
|
fname = f"score{label}.data"
|
|
url = f"https://{self.env.domain1}:{self.server_port}/{fname}"
|
|
scores[label] = self.download_url(label=label, url=url, count=count)
|
|
return scores
|
|
|
|
def _check_uploads(self, r: ExecResult, count: int):
|
|
error = ""
|
|
if r.exit_code != 0:
|
|
error += f"exit={r.exit_code} "
|
|
if r.exit_code != 0 or len(r.stats) != count:
|
|
error += f"stats={len(r.stats)}/{count} "
|
|
fails = [s for s in r.stats if s["response_code"] != 200]
|
|
if len(fails) > 0:
|
|
error += f"{len(fails)} failed"
|
|
for f in fails:
|
|
error += f'[{f["response_code"]}]'
|
|
return error if len(error) > 0 else None
|
|
|
|
def upload_single(self, url: str, fpath: str, count: int):
|
|
sample_size = count
|
|
count = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
self.info("single...")
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env, silent=self._silent_curl, server_addr=self.server_addr
|
|
)
|
|
r = curl.http_put(
|
|
urls=[url],
|
|
fdata=fpath,
|
|
alpn_proto=self.protocol,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
)
|
|
err = self._check_uploads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s["size_upload"] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return {
|
|
"count": count,
|
|
"samples": sample_size,
|
|
"max-parallel": 1,
|
|
"speed": mean(samples) if len(samples) else -1,
|
|
"errors": errors,
|
|
"stats": RunProfile.AverageStats(profiles) if len(profiles) else {},
|
|
}
|
|
|
|
def upload_serial(self, url: str, fpath: str, count: int):
|
|
sample_size = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
url = f"{url}?id=[0-{count - 1}]"
|
|
self.info("serial...")
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env, silent=self._silent_curl, server_addr=self.server_addr
|
|
)
|
|
r = curl.http_put(
|
|
urls=[url],
|
|
fdata=fpath,
|
|
alpn_proto=self.protocol,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
)
|
|
err = self._check_uploads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s["size_upload"] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return {
|
|
"count": count,
|
|
"samples": sample_size,
|
|
"max-parallel": 1,
|
|
"speed": mean(samples) if len(samples) else -1,
|
|
"errors": errors,
|
|
"stats": RunProfile.AverageStats(profiles) if len(profiles) else {},
|
|
}
|
|
|
|
def upload_parallel(self, url: str, fpath: str, count: int):
|
|
sample_size = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
max_parallel = count
|
|
url = f"{url}?id=[0-{count - 1}]"
|
|
self.info("parallel...")
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env, silent=self._silent_curl, server_addr=self.server_addr
|
|
)
|
|
r = curl.http_put(
|
|
urls=[url],
|
|
fdata=fpath,
|
|
alpn_proto=self.protocol,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
extra_args=["--parallel", "--parallel-max", str(max_parallel)],
|
|
)
|
|
err = self._check_uploads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s["size_upload"] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return {
|
|
"count": count,
|
|
"samples": sample_size,
|
|
"max-parallel": max_parallel,
|
|
"speed": mean(samples) if len(samples) else -1,
|
|
"errors": errors,
|
|
"stats": RunProfile.AverageStats(profiles) if len(profiles) else {},
|
|
}
|
|
|
|
def upload_url(self, label: str, url: str, fpath: str, count: int):
|
|
self.info(f" {count}x{label}: ")
|
|
props = {
|
|
"single": self.upload_single(url=url, fpath=fpath, count=10),
|
|
}
|
|
if count > 1:
|
|
props["serial"] = self.upload_serial(url=url, fpath=fpath, count=count)
|
|
props["parallel"] = self.upload_parallel(url=url, fpath=fpath, count=count)
|
|
self.info("ok.\n")
|
|
return props
|
|
|
|
def uploads(self, count: int, fsizes: List[int]) -> Dict[str, Any]:
|
|
scores = {}
|
|
url = f"https://{self.env.domain2}:{self.server_port}/curltest/put"
|
|
fpaths = {}
|
|
for fsize in fsizes:
|
|
label = self.fmt_size(fsize)
|
|
fname = f"upload{label}.data"
|
|
fpaths[label] = self._make_docs_file(
|
|
docs_dir=self.env.gen_dir, fname=fname, fsize=fsize
|
|
)
|
|
|
|
for label, fpath in fpaths.items():
|
|
scores[label] = self.upload_url(
|
|
label=label, url=url, fpath=fpath, count=count
|
|
)
|
|
return scores
|
|
|
|
def do_requests(self, url: str, count: int, max_parallel: int = 1):
|
|
sample_size = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
url = f"{url}?[0-{count - 1}]"
|
|
extra_args = [
|
|
"-w",
|
|
"%{response_code},\\n",
|
|
]
|
|
if max_parallel > 1:
|
|
extra_args.extend(["--parallel", "--parallel-max", str(max_parallel)])
|
|
self.info(f"{max_parallel}...")
|
|
for _ in range(sample_size):
|
|
curl = CurlClient(
|
|
env=self.env, silent=self._silent_curl, server_addr=self.server_addr
|
|
)
|
|
r = curl.http_download(
|
|
urls=[url],
|
|
alpn_proto=self.protocol,
|
|
no_save=True,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
with_stats=False,
|
|
extra_args=extra_args,
|
|
)
|
|
if r.exit_code != 0:
|
|
errors.append(f"exit={r.exit_code}")
|
|
else:
|
|
samples.append(count / r.duration.total_seconds())
|
|
non_200s = 0
|
|
for line in r.stdout.splitlines():
|
|
if not line.startswith("200,"):
|
|
non_200s += 1
|
|
if non_200s > 0:
|
|
errors.append(f"responses != 200: {non_200s}")
|
|
profiles.append(r.profile)
|
|
return {
|
|
"count": count,
|
|
"samples": sample_size,
|
|
"speed": mean(samples) if len(samples) else -1,
|
|
"errors": errors,
|
|
"stats": RunProfile.AverageStats(profiles),
|
|
}
|
|
|
|
def requests_url(self, url: str, count: int):
|
|
self.info(f" {url}: ")
|
|
props = {}
|
|
# 300 is max in curl, see tool_main.h
|
|
for m in [1, 6, 25, 50, 100, 300]:
|
|
props[str(m)] = self.do_requests(url=url, count=count, max_parallel=m)
|
|
self.info("ok.\n")
|
|
return props
|
|
|
|
def requests(self, req_count) -> Dict[str, Any]:
|
|
url = f"https://{self.env.domain1}:{self.server_port}/reqs10.data"
|
|
return {
|
|
"count": req_count,
|
|
"10KB": self.requests_url(url=url, count=req_count),
|
|
}
|
|
|
|
def score(
|
|
self,
|
|
handshakes: bool = True,
|
|
downloads: Optional[List[int]] = None,
|
|
download_count: int = 50,
|
|
uploads: Optional[List[int]] = None,
|
|
upload_count: int = 50,
|
|
req_count=5000,
|
|
requests: bool = True,
|
|
):
|
|
self.info(f"scoring {self.protocol} against {self.server_descr}\n")
|
|
p = {}
|
|
if self.protocol == "h3":
|
|
p["name"] = "h3"
|
|
if not self.env.have_h3_curl():
|
|
raise ScoreCardError("curl does not support HTTP/3")
|
|
for lib in ["ngtcp2", "quiche", "msh3", "nghttp3"]:
|
|
if self.env.curl_uses_lib(lib):
|
|
p["implementation"] = lib
|
|
break
|
|
elif self.protocol == "h2":
|
|
p["name"] = "h2"
|
|
if not self.env.have_h2_curl():
|
|
raise ScoreCardError("curl does not support HTTP/2")
|
|
for lib in ["nghttp2"]:
|
|
if self.env.curl_uses_lib(lib):
|
|
p["implementation"] = lib
|
|
break
|
|
elif self.protocol == "h1" or self.protocol == "http/1.1":
|
|
proto = "http/1.1"
|
|
p["name"] = proto
|
|
p["implementation"] = "native"
|
|
else:
|
|
raise ScoreCardError(f"unknown protocol: {self.protocol}")
|
|
|
|
if "implementation" not in p:
|
|
raise ScoreCardError(f"did not recognized {p} lib")
|
|
p["version"] = Env.curl_lib_version(p["implementation"])
|
|
|
|
score = {
|
|
"curl": self.env.curl_fullname(),
|
|
"os": self.env.curl_os(),
|
|
"protocol": p,
|
|
"server": self.server_descr,
|
|
}
|
|
if handshakes:
|
|
score["handshakes"] = self.handshakes()
|
|
if downloads and len(downloads) > 0:
|
|
score["downloads"] = self.downloads(count=download_count, fsizes=downloads)
|
|
if uploads and len(uploads) > 0:
|
|
score["uploads"] = self.uploads(count=upload_count, fsizes=uploads)
|
|
if requests:
|
|
score["requests"] = self.requests(req_count=req_count)
|
|
self.info("\n")
|
|
return score
|
|
|
|
def fmt_ms(self, tval):
|
|
return f"{int(tval*1000)} ms" if tval >= 0 else "--"
|
|
|
|
def fmt_size(self, val):
|
|
if val >= (1024 * 1024 * 1024):
|
|
return f"{val / (1024*1024*1024):0.000f}GB"
|
|
elif val >= (1024 * 1024):
|
|
return f"{val / (1024*1024):0.000f}MB"
|
|
elif val >= 1024:
|
|
return f"{val / 1024:0.000f}KB"
|
|
else:
|
|
return f"{val:0.000f}B"
|
|
|
|
def fmt_mbs(self, val):
|
|
return f"{val/(1024*1024):0.000f} MB/s" if val >= 0 else "--"
|
|
|
|
def fmt_reqs(self, val):
|
|
return f"{val:0.000f} r/s" if val >= 0 else "--"
|
|
|
|
def print_score(self, score):
|
|
print(f'{score["protocol"]["name"].upper()} in {score["curl"]}')
|
|
if "handshakes" in score:
|
|
print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}')
|
|
print(
|
|
f' {"Host":<17} {"Connect":>12} {"Handshake":>12} '
|
|
f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}'
|
|
)
|
|
for key, val in score["handshakes"].items():
|
|
print(
|
|
f' {key:<17} {self.fmt_ms(val["ipv4-connect"]):>12} '
|
|
f'{self.fmt_ms(val["ipv4-handshake"]):>12} '
|
|
f'{self.fmt_ms(val["ipv6-connect"]):>12} '
|
|
f'{self.fmt_ms(val["ipv6-handshake"]):>12} '
|
|
f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}'
|
|
)
|
|
if "downloads" in score:
|
|
# get the key names of all sizes and measurements made
|
|
sizes = []
|
|
measures = []
|
|
m_names = {}
|
|
mcol_width = 12
|
|
mcol_sw = 17
|
|
for sskey, ssval in score["downloads"].items():
|
|
if isinstance(ssval, str):
|
|
continue
|
|
if sskey not in sizes:
|
|
sizes.append(sskey)
|
|
for mkey, mval in score["downloads"][sskey].items():
|
|
if mkey not in measures:
|
|
measures.append(mkey)
|
|
m_names[
|
|
mkey
|
|
] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})'
|
|
print(f'Downloads from {score["server"]}')
|
|
print(f' {"Size":>8}', end="")
|
|
for m in measures:
|
|
print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end="")
|
|
print(f' {"Errors":^20}')
|
|
|
|
for size in score["downloads"]:
|
|
size_score = score["downloads"][size]
|
|
print(f" {size:>8}", end="")
|
|
errors = []
|
|
for val in size_score.values():
|
|
if "errors" in val:
|
|
errors.extend(val["errors"])
|
|
for m in measures:
|
|
if m in size_score:
|
|
print(
|
|
f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}',
|
|
end="",
|
|
)
|
|
s = (
|
|
f'[{size_score[m]["stats"]["cpu"]:>.1f}%'
|
|
f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]'
|
|
)
|
|
print(f" {s:<{mcol_sw}}", end="")
|
|
else:
|
|
print(" " * mcol_width, end="")
|
|
if len(errors):
|
|
print(f' {"/".join(errors):<20}')
|
|
else:
|
|
print(f' {"-":^20}')
|
|
|
|
if "uploads" in score:
|
|
# get the key names of all sizes and measurements made
|
|
sizes = []
|
|
measures = []
|
|
m_names = {}
|
|
mcol_width = 12
|
|
mcol_sw = 17
|
|
for sskey, ssval in score["uploads"].items():
|
|
if isinstance(ssval, str):
|
|
continue
|
|
if sskey not in sizes:
|
|
sizes.append(sskey)
|
|
for mkey, mval in ssval.items():
|
|
if mkey not in measures:
|
|
measures.append(mkey)
|
|
m_names[
|
|
mkey
|
|
] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})'
|
|
|
|
print(f'Uploads to {score["server"]}')
|
|
print(f' {"Size":>8}', end="")
|
|
for m in measures:
|
|
print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end="")
|
|
print(f' {"Errors":^20}')
|
|
|
|
for size in sizes:
|
|
size_score = score["uploads"][size]
|
|
print(f" {size:>8}", end="")
|
|
errors = []
|
|
for val in size_score.values():
|
|
if "errors" in val:
|
|
errors.extend(val["errors"])
|
|
for m in measures:
|
|
if m in size_score:
|
|
print(
|
|
f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}',
|
|
end="",
|
|
)
|
|
stats = size_score[m]["stats"]
|
|
if "cpu" in stats:
|
|
s = f'[{stats["cpu"]:>.1f}%/{self.fmt_size(stats["rss"])}]'
|
|
else:
|
|
s = "[???/???]"
|
|
print(f" {s:<{mcol_sw}}", end="")
|
|
else:
|
|
print(" " * mcol_width, end="")
|
|
if len(errors):
|
|
print(f' {"/".join(errors):<20}')
|
|
else:
|
|
print(f' {"-":^20}')
|
|
|
|
if "requests" in score:
|
|
sizes = []
|
|
measures = []
|
|
m_names = {}
|
|
mcol_width = 9
|
|
mcol_sw = 13
|
|
for sskey, ssval in score["requests"].items():
|
|
if isinstance(ssval, (str, int)):
|
|
continue
|
|
if sskey not in sizes:
|
|
sizes.append(sskey)
|
|
for mkey in score["requests"][sskey]:
|
|
if mkey not in measures:
|
|
measures.append(mkey)
|
|
m_names[mkey] = f"{mkey}"
|
|
|
|
print('Requests (max parallel) to {score["server"]}')
|
|
print(f' {"Size":>6} {"Reqs":>6}', end="")
|
|
for m in measures:
|
|
print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end="")
|
|
print(f' {"Errors":^10}')
|
|
|
|
for size in sizes:
|
|
size_score = score["requests"][size]
|
|
count = score["requests"]["count"]
|
|
print(f" {size:>6} {count:>6}", end="")
|
|
errors = []
|
|
for val in size_score.values():
|
|
if "errors" in val:
|
|
errors.extend(val["errors"])
|
|
for m in measures:
|
|
if m in size_score:
|
|
print(
|
|
f' {self.fmt_reqs(size_score[m]["speed"]):>{mcol_width}}',
|
|
end="",
|
|
)
|
|
s = (
|
|
f'[{size_score[m]["stats"]["cpu"]:>.1f}%'
|
|
f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]'
|
|
)
|
|
print(f" {s:<{mcol_sw}}", end="")
|
|
else:
|
|
print(" " * mcol_width, end="")
|
|
if len(errors):
|
|
print(f' {"/".join(errors):<10}')
|
|
else:
|
|
print(f' {"-":^10}')
|
|
|
|
|
|
def parse_size(s):
|
|
m = re.match(r"(\d+)(mb|kb|gb)?", s, re.IGNORECASE)
|
|
if m is None:
|
|
raise Exception(f"unrecognized size: {s}")
|
|
size = int(m.group(1))
|
|
if not m.group(2):
|
|
pass
|
|
elif m.group(2).lower() == "kb":
|
|
size *= 1024
|
|
elif m.group(2).lower() == "mb":
|
|
size *= 1024 * 1024
|
|
elif m.group(2).lower() == "gb":
|
|
size *= 1024 * 1024 * 1024
|
|
return size
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
prog="scorecard",
|
|
description="""
|
|
Run a range of tests to give a scorecard for a HTTP protocol
|
|
'h3' or 'h2' implementation in curl.
|
|
""",
|
|
)
|
|
parser.add_argument(
|
|
"-v", "--verbose", action="count", default=1, help="log more output on stderr"
|
|
)
|
|
parser.add_argument(
|
|
"-j",
|
|
"--json",
|
|
action="store_true",
|
|
default=False,
|
|
help="print json instead of text",
|
|
)
|
|
parser.add_argument(
|
|
"-H",
|
|
"--handshakes",
|
|
action="store_true",
|
|
default=False,
|
|
help="evaluate handshakes only",
|
|
)
|
|
parser.add_argument(
|
|
"-d",
|
|
"--downloads",
|
|
action="store_true",
|
|
default=False,
|
|
help="evaluate downloads",
|
|
)
|
|
parser.add_argument(
|
|
"--download",
|
|
action="append",
|
|
type=str,
|
|
default=None,
|
|
help="evaluate download size",
|
|
)
|
|
parser.add_argument(
|
|
"--download-count",
|
|
action="store",
|
|
type=int,
|
|
default=50,
|
|
help="perform that many downloads",
|
|
)
|
|
parser.add_argument(
|
|
"--download-parallel",
|
|
action="store",
|
|
type=int,
|
|
default=0,
|
|
help="perform that many downloads in parallel (default all)",
|
|
)
|
|
parser.add_argument(
|
|
"-u", "--uploads", action="store_true", default=False, help="evaluate uploads"
|
|
)
|
|
parser.add_argument(
|
|
"--upload", action="append", type=str, default=None, help="evaluate upload size"
|
|
)
|
|
parser.add_argument(
|
|
"--upload-count",
|
|
action="store",
|
|
type=int,
|
|
default=50,
|
|
help="perform that many uploads",
|
|
)
|
|
parser.add_argument(
|
|
"-r", "--requests", action="store_true", default=False, help="evaluate requests"
|
|
)
|
|
parser.add_argument(
|
|
"--request-count",
|
|
action="store",
|
|
type=int,
|
|
default=5000,
|
|
help="perform that many requests",
|
|
)
|
|
parser.add_argument(
|
|
"--httpd", action="store_true", default=False, help="evaluate httpd server only"
|
|
)
|
|
parser.add_argument(
|
|
"--caddy", action="store_true", default=False, help="evaluate caddy server only"
|
|
)
|
|
parser.add_argument(
|
|
"--curl-verbose", action="store_true", default=False, help="run curl with `-v`"
|
|
)
|
|
parser.add_argument(
|
|
"protocol", default="h2", nargs="?", help="Name of protocol to score"
|
|
)
|
|
parser.add_argument(
|
|
"--start-only",
|
|
action="store_true",
|
|
default=False,
|
|
help="only start the servers",
|
|
)
|
|
parser.add_argument(
|
|
"--remote",
|
|
action="store",
|
|
type=str,
|
|
default=None,
|
|
help="score against the remote server at <ip>:<port>",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if args.verbose > 0:
|
|
console = logging.StreamHandler()
|
|
console.setLevel(logging.INFO)
|
|
console.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
|
|
logging.getLogger("").addHandler(console)
|
|
|
|
protocol = args.protocol
|
|
handshakes = True
|
|
downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
|
|
if args.download is not None:
|
|
downloads = []
|
|
for x in args.download:
|
|
downloads.extend([parse_size(s) for s in x.split(",")])
|
|
|
|
uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
|
|
if args.upload is not None:
|
|
uploads = []
|
|
for x in args.upload:
|
|
uploads.extend([parse_size(s) for s in x.split(",")])
|
|
|
|
requests = True
|
|
if args.downloads or args.uploads or args.requests or args.handshakes:
|
|
handshakes = args.handshakes
|
|
if not args.downloads:
|
|
downloads = None
|
|
if not args.uploads:
|
|
uploads = None
|
|
requests = args.requests
|
|
|
|
test_httpd = protocol != "h3"
|
|
test_caddy = True
|
|
if args.caddy or args.httpd:
|
|
test_caddy = args.caddy
|
|
test_httpd = args.httpd
|
|
|
|
rv = 0
|
|
env = Env()
|
|
env.setup()
|
|
env.test_timeout = None
|
|
httpd = None
|
|
nghttpx = None
|
|
caddy = None
|
|
try:
|
|
cards = []
|
|
|
|
if args.remote:
|
|
m = re.match(r"^(.+):(\d+)$", args.remote)
|
|
if m is None:
|
|
raise ScoreCardError(
|
|
f"unable to parse ip:port from --remote {args.remote}"
|
|
)
|
|
test_httpd = False
|
|
test_caddy = False
|
|
remote_addr = m.group(1)
|
|
remote_port = int(m.group(2))
|
|
card = ScoreCard(
|
|
env=env,
|
|
protocol=protocol,
|
|
server_descr=f"Server at {args.remote}",
|
|
server_addr=remote_addr,
|
|
server_port=remote_port,
|
|
verbose=args.verbose,
|
|
curl_verbose=args.curl_verbose,
|
|
download_parallel=args.download_parallel,
|
|
)
|
|
cards.append(card)
|
|
|
|
if test_httpd:
|
|
httpd = Httpd(env=env)
|
|
assert httpd.exists(), f"httpd not found: {env.httpd}"
|
|
httpd.clear_logs()
|
|
server_docs = httpd.docs_dir
|
|
assert httpd.start()
|
|
if protocol == "h3":
|
|
nghttpx = NghttpxQuic(env=env)
|
|
nghttpx.clear_logs()
|
|
assert nghttpx.start()
|
|
server_descr = f"nghttpx: https:{env.h3_port} [backend httpd: {env.httpd_version()}, https:{env.https_port}]"
|
|
server_port = env.h3_port
|
|
else:
|
|
server_descr = f"httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}"
|
|
server_port = env.https_port
|
|
card = ScoreCard(
|
|
env=env,
|
|
protocol=protocol,
|
|
server_descr=server_descr,
|
|
server_port=server_port,
|
|
verbose=args.verbose,
|
|
curl_verbose=args.curl_verbose,
|
|
download_parallel=args.download_parallel,
|
|
)
|
|
card.setup_resources(server_docs, downloads)
|
|
cards.append(card)
|
|
|
|
if test_caddy and env.caddy:
|
|
backend = ""
|
|
if uploads and httpd is None:
|
|
backend = f" [backend httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}]"
|
|
httpd = Httpd(env=env)
|
|
assert httpd.exists(), f"httpd not found: {env.httpd}"
|
|
httpd.clear_logs()
|
|
assert httpd.start()
|
|
caddy = Caddy(env=env)
|
|
caddy.clear_logs()
|
|
assert caddy.start()
|
|
server_descr = f"Caddy: {env.caddy_version()}, http:{env.caddy_http_port} https:{env.caddy_https_port}{backend}"
|
|
server_port = caddy.port
|
|
server_docs = caddy.docs_dir
|
|
card = ScoreCard(
|
|
env=env,
|
|
protocol=protocol,
|
|
server_descr=server_descr,
|
|
server_port=server_port,
|
|
verbose=args.verbose,
|
|
curl_verbose=args.curl_verbose,
|
|
download_parallel=args.download_parallel,
|
|
)
|
|
card.setup_resources(server_docs, downloads)
|
|
cards.append(card)
|
|
|
|
if args.start_only:
|
|
print("started servers:")
|
|
for card in cards:
|
|
print(f"{card.server_descr}")
|
|
sys.stderr.write("press [RETURN] to finish")
|
|
sys.stderr.flush()
|
|
sys.stdin.readline()
|
|
else:
|
|
for card in cards:
|
|
score = card.score(
|
|
handshakes=handshakes,
|
|
downloads=downloads,
|
|
download_count=args.download_count,
|
|
uploads=uploads,
|
|
upload_count=args.upload_count,
|
|
req_count=args.request_count,
|
|
requests=requests,
|
|
)
|
|
if args.json:
|
|
print(json.JSONEncoder(indent=2).encode(score))
|
|
else:
|
|
card.print_score(score)
|
|
|
|
except ScoreCardError as ex:
|
|
sys.stderr.write(f"ERROR: {ex}\n")
|
|
rv = 1
|
|
except KeyboardInterrupt:
|
|
log.warning("aborted")
|
|
rv = 1
|
|
finally:
|
|
if caddy:
|
|
caddy.stop()
|
|
if nghttpx:
|
|
nghttpx.stop(wait_dead=False)
|
|
if httpd:
|
|
httpd.stop()
|
|
sys.exit(rv)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|