diff --git a/test/README.pytest b/test/README.pytest index 474030bdc88..d4d73e1c402 100644 --- a/test/README.pytest +++ b/test/README.pytest @@ -35,6 +35,58 @@ also raises the error log level of the tested modules. > pytest -vvv -k test_h2_004_01 run the specific test with mod_http2 at log level TRACE2. +There is an option to archive the results across different +modules and httpd versions. The archiving will preserve +error_log and access_log (cumulative for all tests in the module), +and save a separate config file for each test case. +> pytest -k test_core --archive=/path/to/archive/ + +If you don't provide any specific module, pytest will execute all of them +and the already archived folders and files will be replaced by the new ones. +> pytest --archive=/path/to/archive + +Always use --archive= (with =, not a space) to avoid pytest rootdir issues. + +Using the --archive option having installed a different httpd version +will preserve any results from previous executions so you would +end up with a different folder structure for each httpd version. +You can also archive the results of different modules and that will +work incrementally, which means it will add a new folder per module to the structure. + + +What gets archived: +- error_log and access_log for the module run +- per-test configs: test.conf after each test runs +- module configs: modules.conf, stop.conf +- full server directory (conf, logs, htdocs) +- shared infrastructure (CA, mod_md store, pebble files) in _shared/ to avoid duplication + +Example archive structure with 2 httpd versions: + ├── /path/to/archive + │ ├── 2.4.65 + │ │ ├── _shared + │ │ │ ├── conf + │ │ │ │ ├── httpd.conf + │ │ │ │ └── mime.types + │ │ │ ├── ca + │ │ │ ├── md + │ │ │ ├── acme-ca.pem + │ │ │ └── eab.json + │ │ ├── core + │ │ │ ├── conf + │ │ │ │ ├── modules.conf + │ │ │ │ ├── test_core_001_01.conf + │ │ │ │ ├── test_core_002_01.conf + │ │ │ │ └── ... + │ │ │ ├── logs + │ │ │ │ ├── error_log + │ │ │ │ └── access_log + │ │ │ └── htdocs + │ │ ├── http1 + │ │ └── http2 + │ └── 2.4.66 + │ └── core + By default, test cases will configure httpd with mpm_event. You can change that with the invocation: > MPM=worker pytest test/modules/http2 diff --git a/test/conftest.py b/test/conftest.py index ac0a7c553d8..b3891c17499 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,6 @@ import sys import os +import warnings import pytest @@ -15,6 +16,8 @@ def pytest_addoption(parser): parser.addoption("--repeat", action="store", type=int, default=1, help='Number of times to repeat each test') parser.addoption("--all", action="store_true") + parser.addoption("--archive", action="store", default=None, + help='Archive the server directory after each test package to the specified folder') def pytest_generate_tests(metafunc): @@ -29,6 +32,12 @@ def _function_scope(env, request): env.set_current_test_name(request.node.name) yield env.check_error_log() + archive_dir = request.config.getoption("--archive") + if archive_dir: + fspath = str(request.fspath) + if 'modules/' in fspath: + package_name = fspath.split('modules/')[1].split('/')[0] + env.archive_test_conf(request.node.name, package_name, archive_dir) env.set_current_test_name(None) @@ -39,9 +48,18 @@ def _module_scope(env): @pytest.fixture(autouse=True, scope="package") -def _package_scope(env): +def _package_scope(env, request): env.httpd_error_log.clear_ignored_matches() env.httpd_error_log.clear_ignored_lognos() yield assert env.apache_stop() == 0 env.check_error_log() + + archive_dir = request.config.getoption("--archive") + if archive_dir == "": + warnings.warn("--archive option was empty, skipping archiving") + if archive_dir: + fspath = str(request.fspath) + parts = fspath.split('modules/') + package_name = parts[1].split('/')[0] + env.archive_logs(package_name, archive_dir) diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py index 7c2eb33dacb..c641748ced5 100644 --- a/test/pyhttpd/env.py +++ b/test/pyhttpd/env.py @@ -1,3 +1,4 @@ +import glob import importlib import inspect import logging @@ -20,7 +21,6 @@ from .nghttp import Nghttp from .result import ExecResult - log = logging.getLogger(__name__) @@ -29,7 +29,6 @@ class Dummy: class HttpdTestSetup: - # the modules we want to load MODULES = [ "log_config", @@ -209,8 +208,8 @@ def _build_clients(self): class HttpdTestEnv: - LIBEXEC_DIR = None + SHARED_SERVER_ENTRIES = {'ca', 'md', 'acme-ca.pem', 'eab.json'} @classmethod def has_python_package(cls, name: str) -> bool: @@ -339,9 +338,60 @@ def setup_httpd(self, setup: HttpdTestSetup = None): def check_error_log(self): errors, warnings = self._error_log.get_missed() - assert (len(errors), len(warnings)) == (0, 0),\ - f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ - "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) + assert (len(errors), len(warnings)) == (0, 0), \ + f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n" \ + "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) + + # archives preserving metadata and avoiding duplication + def archive_logs(self, package_name, archive_dir): + version = self.get_httpd_version() + dest = os.path.join(archive_dir, version, package_name) + if os.path.isdir(dest): + shutil.rmtree(dest) + + # use ignore argument to avoid duplication of files + shutil.copytree(self._server_dir, dest, ignore=self.ignore_files) + shared_dest = os.path.join(archive_dir, version, 'shared') + + if os.path.isdir(shared_dest): + shutil.rmtree(shared_dest) + os.makedirs(shared_dest) + + for entry in self.SHARED_SERVER_ENTRIES: + src = os.path.join(self._server_dir, entry) + entry_dest = os.path.join(shared_dest, entry) + if os.path.isdir(src): + shutil.copytree(src, entry_dest) + elif os.path.isfile(src): + shutil.copy2(src, entry_dest) + + shared_conf_dest = os.path.join(shared_dest, 'conf') + os.makedirs(shared_conf_dest) + + # copy them once + for f in ['httpd.conf', 'mime.types']: + src = os.path.join(self._server_conf_dir, f) + if os.path.isfile(src): + shutil.copy2(src, shared_conf_dest) + + def archive_test_conf(self, test_name, package_name, archive_dir): + version = self.get_httpd_version() + dest = os.path.join(archive_dir, version, package_name, 'conf') + if not os.path.isdir(dest): + os.makedirs(dest) + + test_conf = os.path.join(self._server_conf_dir, 'test.conf') + if os.path.isfile(test_conf): + final_name = test_name.replace('/', '_').replace('\\', '_') + dest_file = os.path.join(dest, f"{final_name}.conf") + shutil.copy(test_conf, dest_file) + + # return files to ignore + def ignore_files(self, d, entries): + ignored = [e for e in entries if e.endswith('.sock') or e in self.SHARED_SERVER_ENTRIES] + if os.path.basename(d) == 'conf': + ignored += ['httpd.conf', 'mime.types'] + return ignored @property def curl(self) -> str: @@ -689,7 +739,7 @@ def apache_restart(self): timeout = timedelta(seconds=10) return 0 if self.is_live(self._http_base, timeout=timeout) else -1 return r.exit_code - + def apache_stop(self): r = self._run_apachectl("stop") if r.exit_code == 0: @@ -778,6 +828,7 @@ def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecRe r = ExecResult(args=[], exit_code=0, stdout=b'', stderr=b'') response = None + def fin_response(response): if response: r.add_response(response) @@ -878,7 +929,7 @@ def curl_protocol_version(self, url, timeout=5, options=None): if r.exit_code == 0 and r.response: return r.response["body"].decode('utf-8').rstrip() return -1 - + def nghttp(self): return Nghttp(self._nghttp, connect_addr=self._httpd_addr, tmp_dir=self.gen_dir, test_name=self._current_test) @@ -920,4 +971,3 @@ def make_data_file(self, indir: str, fname: str, fsize: int) -> str: s = f"{i:09d}-{s}\n" fd.write(s[0:remain]) return fpath -