chore: playwright test stability (#14849)

playwright.config.js

@@ -6,7 +6,7 @@ import { defineConfig, devices } from "@playwright/test";
 export default defineConfig({
   testDir: "./tests",
   forbidOnly: !!process.env.CI,
-  retries: process.env.CI ? 2 : 0,
+  retries: process.env.CI ? 3 : 0,
   timeout: 15000, // 15s (default 30s)
   workers: process.env.CI ? 3 : 4,
   reporter: "html",

tests/config-battery.spec.js

@@ -7,8 +7,8 @@ const CONFIG_GRID_ONLY = "config-grid-only.evcc.yaml";
 test.use({ baseURL: baseUrl() });
 
 test.beforeAll(async () => {
-  await start(CONFIG_GRID_ONLY, "password.sql");
   await startSimulator();
+  await start(CONFIG_GRID_ONLY, "password.sql");
 });
 test.afterAll(async () => {
   await stop();

tests/config-grid.spec.js

@@ -7,8 +7,8 @@ const CONFIG_EMPTY = "config-empty.evcc.yaml";
 test.use({ baseURL: baseUrl() });
 
 test.beforeAll(async () => {
-  await start(CONFIG_EMPTY, "password.sql");
   await startSimulator();
+  await start(CONFIG_EMPTY, "password.sql");
 });
 test.afterAll(async () => {
   await stop();

tests/config-messaging.spec.js

@@ -44,8 +44,8 @@ test.describe("messaging", async () => {
 
     await modal.locator(".monaco-editor .view-line").nth(0).click();
     for (let i = 0; i < 4; i++) {
-      await page.keyboard.press(SELECT_ALL);
-      await page.keyboard.press("Backspace");
+      await page.keyboard.press(SELECT_ALL, { delay: 10 });
+      await page.keyboard.press("Backspace", { delay: 10 });
     }
     await page.keyboard.type("# hello world");
     await page.getByRole("button", { name: "Save" }).click();

tests/config-tariffs.spec.js

@@ -51,24 +51,30 @@ test.describe("tariffs", async () => {
     await page.getByTestId("tariffs").getByRole("button", { name: "edit" }).click();
     const modal = await page.getByTestId("tariffs-modal");
     await expect(modal).toBeVisible();
+    await page.waitForLoadState("networkidle");
 
     // default content
     await expect(modal).toContainText("# currency: EUR");
 
     // clear and enter invalid yaml
     await modal.locator(".monaco-editor .view-line").nth(0).click();
-    await page.keyboard.press(SELECT_ALL);
-    await page.keyboard.press("Backspace");
-    await page.keyboard.press(SELECT_ALL);
-    await page.keyboard.press("Backspace");
+
+    for (let i = 0; i < 4; i++) {
+      await page.keyboard.press(SELECT_ALL, { delay: 10 });
+      await page.keyboard.press("Backspace", { delay: 10 });
+    }
+
     await page.keyboard.type("foo: bar\n");
     await page.getByRole("button", { name: "Save" }).click();
     await expect(modal.getByTestId("error")).toContainText("invalid keys: foo");
 
     // clear and enter valid yaml
     await modal.locator(".monaco-editor .view-line").nth(0).click();
-    await page.keyboard.press(SELECT_ALL);
-    await page.keyboard.press("Backspace");
+    for (let i = 0; i < 4; i++) {
+      await page.keyboard.press(SELECT_ALL, { delay: 10 });
+      await page.keyboard.press("Backspace", { delay: 10 });
+    }
+
     await page.keyboard.type("currency: CHF\n");
     await page.keyboard.type("grid:\n");
     await page.keyboard.type("  type: fixed\n");

tests/modals.spec.js

@@ -57,13 +57,13 @@ test.describe("Basics", async () => {
 
 test.describe("Advanced", async () => {
   test.beforeAll(async () => {
-    await start(simulatorConfig(), "password.sql");
     await startSimulator();
+    await start(simulatorConfig(), "password.sql");
   });
 
   test.afterAll(async () => {
-    await stopSimulator();
     await stop();
+    await stopSimulator();
   });
 
   test("Menu options. All available.", async ({ page }) => {

tests/simulator.js

@@ -5,13 +5,13 @@ import waitOn from "wait-on";
 import axios from "axios";
 import { exec } from "child_process";
 
-function port() {
-  const index = process.env.TEST_PARALLEL_INDEX * 1;
+function workerPort() {
+  const index = process.env.TEST_WORKER_INDEX * 1;
   return 12000 + index;
 }
 
 export function simulatorHost() {
-  return `localhost:${port()}`;
+  return `localhost:${workerPort()}`;
 }
 
 export function simulatorUrl() {
@@ -29,9 +29,11 @@ export function simulatorConfig() {
 }
 
 export async function startSimulator() {
-  console.log("starting simulator");
-  const instance = exec(`npm run simulator -- --port ${port()}`);
-  console.log("exec end");
+  const port = workerPort();
+  console.log("starting simulator", { port });
+  console.log(`wait until port ${port} is available`);
+  await waitOn({ resources: [`tcp:localhost:${port}`], reverse: true });
+  const instance = exec(`npm run simulator -- --port ${port}`);
   instance.stdout.pipe(process.stdout);
   instance.stderr.pipe(process.stderr);
 
@@ -40,11 +42,13 @@ export async function startSimulator() {
       throw new Error("simulator terminated", code);
     }
   });
-  console.log("waiton");
   await waitOn({ resources: [`${simulatorUrl()}/api/state`], log: true });
 }
 
 export async function stopSimulator() {
-  console.log("shutting down simulator");
+  const port = workerPort();
+  console.log("shutting down simulator", { port });
   await axios.post(`${simulatorUrl()}/api/shutdown`);
+  console.log(`wait until port ${port} is closed`);
+  await waitOn({ resources: [`tcp:localhost:${port}`], reverse: true });
 }

tests/simulator/api.js

@@ -12,11 +12,11 @@ let state = {
 
 const stateApiMiddleware = (req, res, next) => {
   if (req.method === "POST" && req.originalUrl === "/api/state") {
-    console.log("POST /api/state", req.body);
+    console.log("[simulator] POST /api/state", req.body);
     state = req.body;
     res.end();
   } else if (req.method === "POST" && req.originalUrl === "/api/shutdown") {
-    console.log("POST /api/shutdown", req.body);
+    console.log("[simulator] POST /api/shutdown", req.body);
     res.end();
     process.exit();
   } else if (req.originalUrl === "/api/state") {
@@ -35,7 +35,7 @@ const openemsMiddleware = (req, res, next) => {
   };
   const endpoint = endpoints[req.originalUrl];
   if (req.method === "GET" && endpoint) {
-    console.log("GET", req.originalUrl, endpoint);
+    console.log("[simulator] GET", req.originalUrl, endpoint);
     res.end(JSON.stringify(endpoint));
   } else {
     next();
@@ -46,7 +46,7 @@ export default () => ({
   name: "api",
   enforce: "pre",
   configureServer(server) {
-    console.log("configureServer");
+    console.log("[simulator] configured");
     return () => {
       server.middlewares.use(bodyParser.json());
       server.middlewares.use(stateApiMiddleware);

tests/smart-cost.spec.js

@@ -5,12 +5,12 @@ import { startSimulator, stopSimulator, simulatorUrl, simulatorConfig } from "./
 test.use({ baseURL: baseUrl() });
 
 test.beforeAll(async () => {
-  await start(simulatorConfig(), "password.sql");
   await startSimulator();
+  await start(simulatorConfig(), "password.sql");
 });
 test.afterAll(async () => {
-  await stopSimulator();
   await stop();
+  await stopSimulator();
 });
 
 test.beforeEach(async ({ page }) => {