diff --git a/Makefile b/Makefile index a22f9a03a1..06ed78b665 100644 --- a/Makefile +++ b/Makefile @@ -254,11 +254,23 @@ $(eval $(call install-sh,standard,operator-controller-standard.yaml)) .PHONY: test test: manifests generate fmt lint test-unit test-e2e test-regression #HELP Run all tests. -E2E_TIMEOUT ?= 20m -GODOG_ARGS ?= .PHONY: e2e +e2e: E2E_TIMEOUT ?= 20m +e2e: GODOG_ARGS ?= e2e: #EXHELP Run the e2e tests. - go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) $(if $(GODOG_ARGS),-args $(GODOG_ARGS)) +ifeq ($(strip $(GODOG_ARGS)),) + set +e; \ + go test -count=1 -v ./test/e2e/features_test.go -timeout=${E2E_TIMEOUT} -args --godog.tags="~@Serial" --godog.concurrency=100; \ + parallelExit=$$?; \ + go test -count=1 -v ./test/e2e/features_test.go -timeout=${E2E_TIMEOUT} -args --godog.tags="@Serial" --godog.concurrency=1; \ + serialExit=$$?; \ + if [[ $$parallelExit -ne 0 ]] || [[ $$serialExit -ne 0 ]]; then \ + echo "e2e tests failed: parallel=$$parallelExit serial=$$serialExit"; \ + exit 1; \ + fi +else + go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) -args $(GODOG_ARGS) +endif export CLUSTER_REGISTRY_HOST := docker-registry.operator-controller-e2e.svc:5000 .PHONY: extension-developer-e2e @@ -316,7 +328,7 @@ test-experimental-e2e: COVERAGE_NAME := experimental-e2e test-experimental-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) test-experimental-e2e: export INSTALL_DEFAULT_CATALOGS := false test-experimental-e2e: PROMETHEUS_VALUES := helm/prom_experimental.yaml -test-experimental-e2e: E2E_TIMEOUT := 25m +test-experimental-e2e: E2E_TIMEOUT ?= 25m test-experimental-e2e: run-internal prometheus e2e e2e-coverage kind-clean #HELP Run experimental e2e test suite on local kind cluster .PHONY: prometheus diff --git a/test/e2e/README.md b/test/e2e/README.md index 98b99b9caa..817a5c3832 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -207,6 +207,10 @@ Use these variables in YAML templates: ### 5. Feature Tags +Tags can be used for different purposes in the test suite: + +#### Feature Gate Tags + Use tags to conditionally run scenarios based on feature gates: ```gherkin @@ -216,6 +220,28 @@ Scenario: Install operator having webhooks Scenarios are skipped if the feature gate is not enabled on the deployed controller. +#### Serial Execution Tag + +By default, scenarios run concurrently (up to 100 parallel scenarios). However, some tests must run serially, typically because they: +- Modify shared cluster resources (e.g., cluster-wide TLS configuration) +- Have resource constraints that prevent parallel execution +- Require exclusive access to a resource + +To mark a test for serial execution, add the `@Serial` tag: + +```gherkin +@Serial +Feature: TLS profile enforcement on metrics endpoints + + Scenario: Test TLS configuration + Given the "catalogd" deployment is configured with custom TLS settings + ... +``` + +The `Makefile` automatically separates scenarios when run without additional `GODOG_ARGS`: +- Scenarios **without** `@Serial` run concurrently in the first test phase +- Scenarios **with** `@Serial` run sequentially in a separate serial test phase + ## Running Tests ### Run All Tests @@ -230,6 +256,15 @@ or make test-experimental-e2e ``` +Custom godog arguments can be modified by setting the following: +```bash +GODOG_ARGS=--godog.tags=@WebhookProviderCertManager make test-experimental-e2e +``` + +Note that when this is done the `make` target will no longer automatically split the test run into parallel and serial runs, and test execution time may increase. If you wish to add concurrency back into the arguments, it is recommended to also disable the `@Serial` tests: +```bash +GODOG_ARGS="--godog.tags=~@Serial --godog.concurrency=100" make test-experimental-e2e +``` ### Run Specific Feature diff --git a/test/e2e/features/ha.feature b/test/e2e/features/ha.feature index 6b889c99bb..bb20cfafa8 100644 --- a/test/e2e/features/ha.feature +++ b/test/e2e/features/ha.feature @@ -1,3 +1,4 @@ +@Serial Feature: HA failover for catalogd When catalogd is deployed with multiple replicas, the remaining pods must diff --git a/test/e2e/features/proxy.feature b/test/e2e/features/proxy.feature index c4ddcc091c..9f30a47f0e 100644 --- a/test/e2e/features/proxy.feature +++ b/test/e2e/features/proxy.feature @@ -1,3 +1,4 @@ +@Serial Feature: HTTPS proxy support for outbound catalog requests OLM's operator-controller fetches catalog data from catalogd over HTTPS. diff --git a/test/e2e/features/revision.feature b/test/e2e/features/revision.feature index 866e8195f4..0ad542f4d7 100644 --- a/test/e2e/features/revision.feature +++ b/test/e2e/features/revision.feature @@ -672,6 +672,7 @@ Feature: Install ClusterObjectSet And resource "deployment/test-deployment" is eventually not found @ProgressDeadline + @Serial Scenario: COS recovers from ProgressDeadlineExceeded to Succeeded when probes pass Given min value for ClusterObjectSet .spec.progressDeadlineMinutes is set to 1 And ServiceAccount "olm-sa" with needed permissions is available in test namespace @@ -721,6 +722,7 @@ Feature: Install ClusterObjectSet containers: - name: delayed-ready image: busybox:1.36 + imagePullPolicy: IfNotPresent command: ["sleep", "1000"] readinessProbe: exec: diff --git a/test/e2e/features/tls.feature b/test/e2e/features/tls.feature index 77293c12f3..ba43f74141 100644 --- a/test/e2e/features/tls.feature +++ b/test/e2e/features/tls.feature @@ -1,3 +1,4 @@ +@Serial Feature: TLS profile enforcement on metrics endpoints Background: @@ -6,11 +7,15 @@ Feature: TLS profile enforcement on metrics endpoints # Each scenario patches the deployment with the TLS settings under test and # restores the original configuration during cleanup, so scenarios are independent. + # This feature file is run serially to avoid potential issues modifying the catalogd + # deployment during functional testing. + # All three scenarios test catalogd only: the enforcement logic lives in the shared # tlsprofiles package, so one component is sufficient. TLS 1.2 is used for cipher # and curve enforcement because Go's crypto/tls does not allow the server to restrict # TLS 1.3 cipher suites — CipherSuites config only applies to TLS 1.2. The e2e cert # uses ECDSA, so ECDHE_ECDSA cipher families are required. + @TLSProfile Scenario: catalogd metrics endpoint enforces configured minimum TLS version Given the "catalogd" deployment is configured with custom TLS minimum version "TLSv1.3"