From 70d9481de4c81edfc0a213cbe9e85f49eae612c8 Mon Sep 17 00:00:00 2001
From: Michael Geers <michael@geers.tv>
Date: Wed, 25 Jun 2025 14:52:59 +0200
Subject: [PATCH] chore: playwright stability (#22042)

---
 .../GlobalSettings/GlobalSettingsModal.vue    | 37 +++++-------------
 package-lock.json                             | 38 +++++++++++++++++++
 package.json                                  |  2 +
 playwright.config.js                          |  3 +-
 tests/config-vehicles.spec.ts                 | 14 +++++++
 tests/config.spec.ts                          |  5 ---
 tests/evcc.ts                                 | 32 +++++++++++-----
 tests/modals.spec.ts                          | 19 ++++++++--
 tests/utils.ts                                |  8 +++-
 9 files changed, 108 insertions(+), 50 deletions(-)

diff --git a/assets/js/components/GlobalSettings/GlobalSettingsModal.vue b/assets/js/components/GlobalSettings/GlobalSettingsModal.vue
index 2f102d7b8..d7545af1f 100644
--- a/assets/js/components/GlobalSettings/GlobalSettingsModal.vue
+++ b/assets/js/components/GlobalSettings/GlobalSettingsModal.vue
@@ -1,41 +1,22 @@
 <template>
-	<Teleport to="body">
-		<div
-			id="globalSettingsModal"
-			class="modal fade text-dark"
-			data-bs-backdrop="true"
-			tabindex="-1"
-			role="dialog"
-			aria-hidden="true"
-		>
-			<div class="modal-dialog modal-dialog-centered" role="document">
-				<div class="modal-content">
-					<div class="modal-header">
-						<h5 class="modal-title">{{ $t("settings.title") }}</h5>
-						<button
-							type="button"
-							class="btn-close"
-							data-bs-dismiss="modal"
-							aria-label="Close"
-						></button>
-					</div>
-					<div class="modal-body">
-						<UserInterfaceSettings :sponsor="sponsor" />
-					</div>
-				</div>
-			</div>
-		</div>
-	</Teleport>
+	<GenericModal
+		id="globalSettingsModal"
+		:title="$t('settings.title')"
+		data-testid="global-settings-modal"
+	>
+		<UserInterfaceSettings :sponsor="sponsor" />
+	</GenericModal>
 </template>
 
 <script lang="ts">
 import { defineComponent, type PropType } from "vue";
+import GenericModal from "../Helper/GenericModal.vue";
 import UserInterfaceSettings from "./UserInterfaceSettings.vue";
 import type { Sponsor } from "@/types/evcc";
 
 export default defineComponent({
 	name: "GlobalSettingsModal",
-	components: { UserInterfaceSettings },
+	components: { GenericModal, UserInterfaceSettings },
 	props: {
 		sponsor: { type: Object as PropType<Sponsor>, default: () => ({}) },
 	},
diff --git a/package-lock.json b/package-lock.json
index 8e02a417c..6627744eb 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -41,6 +41,7 @@
         "eslint-plugin-vue": "^9.3.0",
         "globals": "^16.0.0",
         "happy-dom": "^17.4.4",
+        "kill-port": "^2.0.1",
         "monaco-editor": "^0.52.2",
         "mqtt": "^5.11.0",
         "prettier": "^3.0.0",
@@ -70,6 +71,7 @@
         "@storybook/vue3": "^8.6.4",
         "@storybook/vue3-vite": "^8.6.4",
         "@types/body-parser": "^1.19.6",
+        "@types/kill-port": "^2.0.3",
         "@types/smoothscroll-polyfill": "^0.3.4",
         "@types/wait-on": "^5.3.4",
         "cross-env": "^7.0.3",
@@ -3447,6 +3449,17 @@
       "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
       "license": "MIT"
     },
+    "node_modules/@types/kill-port": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@types/kill-port/-/kill-port-2.0.3.tgz",
+      "integrity": "sha512-ZHs59e5FBjDLQLOxM48+814LSyNf5sgpi0odtJ0FH6xrIAZXb4yksYG+4mZCbidX3fBOfHytAKAVMgkWvv/Piw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*",
+        "shell-exec": "^1"
+      }
+    },
     "node_modules/@types/mdx": {
       "version": "2.0.13",
       "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.13.tgz",
@@ -6693,6 +6706,12 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/get-them-args": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/get-them-args/-/get-them-args-1.3.2.tgz",
+      "integrity": "sha512-LRn8Jlk+DwZE4GTlDbT3Hikd1wSHgLMme/+7ddlqKd7ldwR6LjJgTVWzBnR01wnYGe4KgrXjg287RaI22UHmAw==",
+      "license": "MIT"
+    },
     "node_modules/glob": {
       "version": "10.4.5",
       "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
@@ -7696,6 +7715,19 @@
         "json-buffer": "3.0.1"
       }
     },
+    "node_modules/kill-port": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/kill-port/-/kill-port-2.0.1.tgz",
+      "integrity": "sha512-e0SVOV5jFo0mx8r7bS29maVWp17qGqLBZ5ricNSajON6//kmb7qqqNnml4twNE8Dtj97UQD+gNFOaipS/q1zzQ==",
+      "license": "MIT",
+      "dependencies": {
+        "get-them-args": "1.3.2",
+        "shell-exec": "1.0.2"
+      },
+      "bin": {
+        "kill-port": "cli.js"
+      }
+    },
     "node_modules/levn": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@@ -9336,6 +9368,12 @@
         "node": ">=8"
       }
     },
+    "node_modules/shell-exec": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/shell-exec/-/shell-exec-1.0.2.tgz",
+      "integrity": "sha512-jyVd+kU2X+mWKMmGhx4fpWbPsjvD53k9ivqetutVW/BQ+WIZoDoP4d8vUMGezV6saZsiNoW2f9GIhg9Dondohg==",
+      "license": "MIT"
+    },
     "node_modules/side-channel": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
diff --git a/package.json b/package.json
index b2027bcf8..b1bda6068 100644
--- a/package.json
+++ b/package.json
@@ -51,6 +51,7 @@
     "eslint-plugin-vue": "^9.3.0",
     "globals": "^16.0.0",
     "happy-dom": "^17.4.4",
+    "kill-port": "^2.0.1",
     "monaco-editor": "^0.52.2",
     "mqtt": "^5.11.0",
     "prettier": "^3.0.0",
@@ -91,6 +92,7 @@
     "@storybook/vue3": "^8.6.4",
     "@storybook/vue3-vite": "^8.6.4",
     "@types/body-parser": "^1.19.6",
+    "@types/kill-port": "^2.0.3",
     "@types/smoothscroll-polyfill": "^0.3.4",
     "@types/wait-on": "^5.3.4",
     "cross-env": "^7.0.3",
diff --git a/playwright.config.js b/playwright.config.js
index f90f74e2b..114cd6dea 100644
--- a/playwright.config.js
+++ b/playwright.config.js
@@ -8,8 +8,7 @@ export default defineConfig({
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 4 : 0,
   timeout: 30000, // default 30s
-  workers: process.env.CI ? 3 : 4,
-  reporter: process.env.CI ? "github" : "list",
+  reporter: [[process.env.CI ? "github" : "list"], ["html", { open: "never" }]],
   use: {
     baseURL: "http://127.0.0.1:7070",
     trace: "on-first-retry",
diff --git a/tests/config-vehicles.spec.ts b/tests/config-vehicles.spec.ts
index 9283d1139..a83fe1884 100644
--- a/tests/config-vehicles.spec.ts
+++ b/tests/config-vehicles.spec.ts
@@ -32,17 +32,21 @@ test.describe("vehicles", async () => {
 
     // create #1
     await page.getByTestId("add-vehicle").click();
+    await expectModalVisible(vehicleModal);
     await vehicleModal.getByLabel("Manufacturer").selectOption(GENERIC_VEHICLE);
     await vehicleModal.getByLabel("Title").fill("Green Car");
     await vehicleModal.getByRole("button", { name: "Validate & save" }).click();
+    await expectModalHidden(vehicleModal);
 
     await expect(page.getByTestId("vehicle")).toHaveCount(1);
 
     // create #2
     await page.getByTestId("add-vehicle").click();
+    await expectModalVisible(vehicleModal);
     await vehicleModal.getByLabel("Manufacturer").selectOption(GENERIC_VEHICLE);
     await vehicleModal.getByLabel("Title").fill("Yellow Van");
     await vehicleModal.getByRole("button", { name: "Validate & save" }).click();
+    await expectModalHidden(vehicleModal);
 
     await expect(page.getByTestId("vehicle")).toHaveCount(2);
     await expect(page.getByTestId("vehicle").nth(0)).toHaveText(/Green Car/);
@@ -87,16 +91,20 @@ test.describe("vehicles", async () => {
 
     // create #1 & #2
     await page.getByTestId("add-vehicle").click();
+    await expectModalVisible(vehicleModal);
     await vehicleModal.getByLabel("Manufacturer").selectOption(GENERIC_VEHICLE);
     await vehicleModal.getByLabel("Title").fill("Green Car");
     await vehicleModal.getByRole("button", { name: "Validate & save" }).click();
+    await expectModalHidden(vehicleModal);
 
     await page.getByTestId("add-vehicle").click();
+    await expectModalVisible(vehicleModal);
     await vehicleModal.getByLabel("Manufacturer").selectOption(GENERIC_VEHICLE);
     await vehicleModal.getByLabel("Title").fill("Yellow Van");
     await vehicleModal.getByLabel("car").click();
     await vehicleModal.getByLabel("van").check();
     await vehicleModal.getByRole("button", { name: "Validate & save" }).click();
+    await expectModalHidden(vehicleModal);
 
     await expect(page.getByTestId("vehicle")).toHaveCount(2);
 
@@ -120,9 +128,11 @@ test.describe("vehicles", async () => {
 
     // create #2
     await page.getByTestId("add-vehicle").click();
+    await expectModalVisible(vehicleModal);
     await vehicleModal.getByLabel("Manufacturer").selectOption(GENERIC_VEHICLE);
     await vehicleModal.getByLabel("Title").fill("Green Car");
     await vehicleModal.getByRole("button", { name: "Validate & save" }).click();
+    await expectModalHidden(vehicleModal);
 
     await expect(page.getByTestId("vehicle")).toHaveCount(2);
     await expect(page.getByTestId("vehicle").nth(0)).toHaveText(/YAML Bike/);
@@ -139,6 +149,7 @@ test.describe("vehicles", async () => {
     const vehicleModal = page.getByTestId("vehicle-modal");
 
     // generic
+    await expectModalVisible(vehicleModal);
     await vehicleModal.getByLabel("Manufacturer").selectOption(GENERIC_VEHICLE);
     await expect(vehicleModal.getByLabel("Title")).toBeVisible();
     await expect(vehicleModal.getByLabel("Car")).toBeVisible(); // icon
@@ -177,11 +188,13 @@ test.describe("vehicles", async () => {
     const vehicleModal = page.getByTestId("vehicle-modal");
 
     // generic
+    await expectModalVisible(vehicleModal);
     await vehicleModal.getByLabel("Manufacturer").selectOption(GENERIC_VEHICLE);
     await vehicleModal.getByLabel("Title").fill("RFID Car");
     await page.getByRole("button", { name: "Show advanced settings" }).click();
     await vehicleModal.getByLabel("RFID identifiers").fill("aaa\nbbb \n ccc\n\nddd\n");
     await vehicleModal.getByRole("button", { name: "Validate & save" }).click();
+    await expectModalHidden(vehicleModal);
     await expect(page.getByTestId("restart-needed")).toBeVisible();
 
     // restart evcc
@@ -194,6 +207,7 @@ test.describe("vehicles", async () => {
     await vehicleModal.getByRole("button", { name: "Show advanced settings" }).click();
     await expect(vehicleModal.getByLabel("RFID identifiers")).toHaveValue("aaa\nbbb\nccc\nddd");
     await vehicleModal.getByLabel("Close").click();
+    await expectModalHidden(vehicleModal);
     await expect(page.getByTestId("fatal-error")).not.toBeVisible();
   });
 
diff --git a/tests/config.spec.ts b/tests/config.spec.ts
index af0271f70..733ead714 100644
--- a/tests/config.spec.ts
+++ b/tests/config.spec.ts
@@ -20,11 +20,6 @@ test.describe("basics", async () => {
     await page.getByRole("link", { name: "Configuration" }).click();
     await expect(page.getByRole("heading", { name: "Configuration" })).toBeVisible();
   });
-  test.skip("alert box should always be visible", async ({ page }) => {
-    await page.goto("/#/config");
-    await enableExperimental(page);
-    await expect(page.getByRole("alert")).toBeVisible();
-  });
 });
 
 test.describe("general", async () => {
diff --git a/tests/evcc.ts b/tests/evcc.ts
index 43453f795..c71fe9925 100644
--- a/tests/evcc.ts
+++ b/tests/evcc.ts
@@ -2,6 +2,7 @@ import fs from "fs";
 import waitOn from "wait-on";
 import axios from "axios";
 import { spawn, execSync, ChildProcess } from "child_process";
+import killPort from "kill-port";
 import os from "os";
 import path from "path";
 import { Transform } from "stream";
@@ -117,18 +118,29 @@ async function _stop(instance?: ChildProcess) {
     log("evcc is down", { port });
     return;
   }
-  // check if auth is required
-  const res = await axios.get(`${baseUrl()}/api/auth/status`);
-  log("auth status", res.status, res.statusText, res.data);
   let cookie;
-  // login required
-  if (!res.data) {
-    const res = await axios.post(`${baseUrl()}/api/auth/login`, { password: "secret" });
-    log("login", res.status, res.statusText);
-    cookie = res.headers["set-cookie"];
+  try {
+    // check if auth is required
+    const res = await axios.get(`${baseUrl()}/api/auth/status`);
+    log("auth status", res.status, res.statusText, res.data);
+    // login required
+    if (!res.data) {
+      const res = await axios.post(`${baseUrl()}/api/auth/login`, { password: "secret" });
+      log("login", res.status, res.statusText);
+      cookie = res.headers["set-cookie"];
+    }
+    log("shutting down evcc", { port });
+    await axios.post(`${baseUrl()}/api/system/shutdown`, {}, { headers: { cookie } });
+  } catch (error) {
+    const port = workerPort();
+    log(`shutdown failed, last resort: kill by port`, port, error);
+    try {
+      await killPort(port);
+      log(`killed process on port ${port}`);
+    } catch (killError) {
+      log(`no process found on port ${port} or kill failed:`, killError);
+    }
   }
-  log("shutting down evcc", { port });
-  await axios.post(`${baseUrl()}/api/system/shutdown`, {}, { headers: { cookie } });
   log(`wait until port ${port} is closed`);
   await waitOn({ resources: [`tcp:${port}`], reverse: true, log: LOG_ENABLED });
   log("evcc is down", { port });
diff --git a/tests/modals.spec.ts b/tests/modals.spec.ts
index 560a77e0e..d956cfd6e 100644
--- a/tests/modals.spec.ts
+++ b/tests/modals.spec.ts
@@ -4,7 +4,11 @@ import { startSimulator, stopSimulator, simulatorConfig } from "./simulator";
 
 const BASICS_CONFIG = "basics.evcc.yaml";
 
-const UI_ROUTES = ["/", "/#/sessions", "/#/config"];
+const UI_ROUTES = [
+  { path: "/" },
+  { path: "/#/sessions", title: "Sessions" },
+  { path: "/#/config", title: "Configuration" },
+];
 
 test.use({ baseURL: baseUrl() });
 
@@ -13,14 +17,17 @@ test.describe("Basics", async () => {
     await start(BASICS_CONFIG);
   });
 
+  const title = "Hello World";
+
   test.afterAll(async () => {
     await stop();
   });
 
   test("Menu options. No battery and grid.", async ({ page }) => {
     for (const route of UI_ROUTES) {
-      await page.goto(route);
+      await page.goto(route.path);
 
+      await expect(page.getByRole("heading", { name: route.title || title })).toBeVisible();
       await page.getByTestId("topnavigation-button").click();
       await expect(page.getByRole("button", { name: "User Interface" })).toBeVisible();
       await expect(page.getByRole("button", { name: "Home Battery" })).not.toBeVisible();
@@ -31,6 +38,7 @@ test.describe("Basics", async () => {
   test("Need help?", async ({ page }) => {
     await page.goto("/");
 
+    await expect(page.getByRole("heading", { name: "Hello World" })).toBeVisible();
     await page.getByTestId("topnavigation-button").click();
     await page.getByRole("button", { name: "Need help?" }).click();
 
@@ -39,6 +47,7 @@ test.describe("Basics", async () => {
 
   test("User Interface", async ({ page }) => {
     await page.goto("/");
+    await expect(page.getByRole("heading", { name: "Hello World" })).toBeVisible();
     await page.getByTestId("topnavigation-button").click();
     await page.getByRole("button", { name: "User Interface" }).click();
 
@@ -57,10 +66,13 @@ test.describe("Advanced", async () => {
     await stopSimulator();
   });
 
+  const title = "evcc Simulator";
+
   test("Menu options. All available.", async ({ page }) => {
     for (const route of UI_ROUTES) {
-      await page.goto(route);
+      await page.goto(route.path);
 
+      await expect(page.getByRole("heading", { name: route.title || title })).toBeVisible();
       await page.getByTestId("topnavigation-button").click();
       await expect(page.getByRole("button", { name: "User Interface" })).toBeVisible();
       await expect(page.getByRole("button", { name: "Home Battery" })).toBeVisible();
@@ -70,6 +82,7 @@ test.describe("Advanced", async () => {
 
   test("Home Battery from top navigation", async ({ page }) => {
     await page.goto("/");
+    await expect(page.getByRole("heading", { name: title })).toBeVisible();
     await page.getByTestId("topnavigation-button").click();
     await page.getByRole("button", { name: "Home Battery" }).click();
 
diff --git a/tests/utils.ts b/tests/utils.ts
index 7588c6cda..60bf7a226 100644
--- a/tests/utils.ts
+++ b/tests/utils.ts
@@ -1,10 +1,14 @@
 import { expect, type Page, type Locator } from "@playwright/test";
 
 export async function enableExperimental(page: Page): Promise<void> {
+  await expect(page.getByTestId("topnavigation-button")).toBeVisible();
   await page.getByTestId("topnavigation-button").click();
   await page.getByTestId("topnavigation-settings").click();
-  await page.getByLabel("Experimental 🧪").click();
-  await page.getByRole("button", { name: "Close" }).click();
+  const modal = page.getByTestId("global-settings-modal");
+  await expectModalVisible(modal);
+  await modal.getByLabel("Experimental 🧪").click();
+  await modal.getByRole("button", { name: "Close" }).click();
+  await expectModalHidden(modal);
   await expect(page.locator(".modal-backdrop")).not.toBeVisible();
 }