API: generate /api/state documentation from UI types (#32431)

Co-authored-by: Michael Geers <michael@geers.tv>

scripts/state-schema/index.ts

@@ -0,0 +1,33 @@
+import { buildSchemas } from "./schemas";
+import { writeStateSchemas, bundleMcpJson } from "./openapi";
+
+const command = process.argv[2] ?? "generate";
+
+switch (command) {
+  case "generate": {
+    const schemas = buildSchemas();
+    writeStateSchemas(schemas);
+    await bundleMcpJson();
+    console.log(
+      `${Object.keys(schemas.defs).length} state schemas → openapi.state.yaml + mcp/openapi.json`
+    );
+    break;
+  }
+  case "dump": {
+    console.log(JSON.stringify(buildSchemas().defs, null, 2));
+    break;
+  }
+  case "validate": {
+    const source = process.argv[3];
+    if (!source) {
+      console.error("usage: vp run openapi -- validate <payload.json | url>");
+      process.exit(1);
+    }
+    const { validate } = await import("./validate");
+    if (!(await validate(source))) process.exit(1);
+    break;
+  }
+  default:
+    console.error(`unknown command: ${command}`);
+    process.exit(1);
+}

scripts/state-schema/openapi.ts

@@ -0,0 +1,38 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { parse, stringify } from "yaml";
+import { bundle, createConfig } from "@redocly/openapi-core";
+import type { StateSchemas } from "./schemas";
+
+const OPENAPI_PATH = "server/openapi.yaml";
+const STATE_PATH = "server/openapi.state.yaml";
+const MCP_JSON_PATH = "server/mcp/openapi.json";
+
+const HEADER = `# GENERATED FILE - DO NOT EDIT (source: assets/js/types/evcc.ts, update: make openapi)
+`;
+
+export function writeStateSchemas(schemas: StateSchemas): void {
+  const root = parse(readFileSync(OPENAPI_PATH, "utf8"));
+  const handWritten = new Set(Object.keys(root.components?.schemas ?? {}));
+  for (const name of Object.keys(schemas.defs)) {
+    if (handWritten.has(name)) {
+      throw new Error(`generated schema "${name}" collides with hand-written component`);
+    }
+  }
+
+  const doc =
+    HEADER +
+    stringify(
+      { components: { schemas: schemas.defs } },
+      { lineWidth: 0, aliasDuplicateObjects: false }
+    );
+
+  writeFileSync(STATE_PATH, doc);
+}
+
+// inline the multi-file spec into the single json embedded by the MCP server
+export async function bundleMcpJson(): Promise<void> {
+  const { bundle: result } = await bundle({ ref: OPENAPI_PATH, config: await createConfig({}) });
+  const doc = result.parsed as { servers?: unknown };
+  delete doc.servers; // mcp server sets its own url
+  writeFileSync(MCP_JSON_PATH, JSON.stringify(doc, null, 2) + "\n");
+}

scripts/state-schema/schemas.ts

@@ -0,0 +1,221 @@
+import { createGenerator } from "ts-json-schema-generator";
+import type { Schema } from "ts-json-schema-generator";
+
+export interface StateSchemas {
+  rootName: string;
+  // schema per component name, root first, rest alphabetical
+  defs: Record<string, Schema>;
+}
+
+// enums use SCREAMING_SNAKE in the frontend, schema names follow openapi PascalCase convention
+const RENAME: Record<string, string> = {
+  CHARGE_MODE: "ChargeMode",
+  BATTERY_MODE: "BatteryMode",
+  CURRENCY: "Currency",
+  CHARGER_STATUS_REASON: "ChargerStatusReason",
+  PHASE_ACTION: "PhaseAction",
+  PV_ACTION: "PvAction",
+  SMART_COST_TYPE: "SmartCostType",
+  OCPP_STATION_STATUS: "OcppConnectionStatus",
+  MODBUS_BAUDRATE: "ModbusBaudrate",
+  MODBUS_COMSET: "ModbusComset",
+  MODBUS_PROXY_READONLY: "ModbusProxyReadonly",
+};
+
+const VALID_NAME = /^[a-zA-Z0-9._-]+$/;
+const ROOT = "State";
+
+function buildRawSchema(): Schema {
+  const generator = createGenerator({
+    path: "assets/js/types/evcc.ts",
+    tsconfig: "tsconfig.json",
+    type: ROOT,
+    jsDoc: "extended",
+    extraTags: ["internal"],
+    additionalProperties: true,
+    topRef: false,
+    skipTypeCheck: true,
+    sortProps: false,
+  });
+  return generator.createSchema(ROOT);
+}
+
+type AnySchema = any;
+
+function walk(node: any, visit: (schema: AnySchema) => void): void {
+  if (Array.isArray(node)) {
+    node.forEach((child) => walk(child, visit));
+  } else if (node && typeof node === "object") {
+    visit(node);
+    Object.values(node).forEach((child) => walk(child, visit));
+  }
+}
+
+// remove properties tagged @internal, they are UI-only and not part of the API payload
+function stripInternal(schema: AnySchema): void {
+  walk(schema, (node) => {
+    if (!node.properties) return;
+    for (const [key, prop] of Object.entries<AnySchema>(node.properties)) {
+      if (prop?.internal === true) {
+        delete node.properties[key];
+        if (Array.isArray(node.required)) {
+          node.required = node.required.filter((r: string) => r !== key);
+          if (node.required.length === 0) delete node.required;
+        }
+      }
+    }
+  });
+}
+
+function collectRefs(schema: AnySchema): Set<string> {
+  const refs = new Set<string>();
+  walk(schema, (node) => {
+    if (typeof node.$ref === "string") {
+      refs.add(decodeURIComponent(node.$ref.replace("#/definitions/", "")));
+    }
+  });
+  return refs;
+}
+
+// drop definitions that became unreachable after stripping @internal properties
+function reachableDefs(
+  root: AnySchema,
+  defs: Record<string, AnySchema>
+): Record<string, AnySchema> {
+  const keep: Record<string, AnySchema> = {};
+  const queue = [...collectRefs(root)];
+  while (queue.length > 0) {
+    const name = queue.shift()!;
+    if (keep[name] || !defs[name]) continue;
+    keep[name] = defs[name];
+    queue.push(...collectRefs(defs[name]));
+  }
+  return keep;
+}
+
+// convert json-schema null unions to `nullable: true`, the 3.0 style used across openapi.yaml
+// (kin-openapi, which validates the spec in CI, does not support 3.1 type arrays)
+function normalizeNullables(schema: AnySchema): void {
+  walk(schema, (node) => {
+    if (Array.isArray(node.type) && node.type.includes("null")) {
+      const rest = node.type.filter((t: string) => t !== "null");
+      if (rest.length !== 1) throw new Error(`unsupported type union ${node.type}`);
+      node.type = rest[0];
+      node.nullable = true;
+    }
+    if (Array.isArray(node.anyOf) && node.anyOf.some((b: AnySchema) => b.type === "null")) {
+      const rest = node.anyOf.filter((b: AnySchema) => b.type !== "null");
+      delete node.anyOf;
+      node.nullable = true;
+      if (rest.length === 1 && !rest[0].$ref) {
+        Object.assign(node, rest[0]);
+      } else {
+        node.allOf = rest;
+      }
+    }
+    // openapi 3.0 has no `const`, use a single-value enum
+    if ("const" in node) {
+      node.enum = [node.const];
+      delete node.const;
+    }
+    // openapi 3.0 uses a singular example
+    if (Array.isArray(node.examples)) {
+      node.example = node.examples[0];
+      delete node.examples;
+    }
+  });
+}
+
+// point $refs at the final openapi component names
+function rewriteRefs(schema: AnySchema, finalNames: Map<string, string>): void {
+  walk(schema, (node) => {
+    if (typeof node.$ref !== "string" || node.$ref.startsWith("#/components/")) return;
+    const name = decodeURIComponent(node.$ref.replace("#/definitions/", ""));
+    const renamed = finalNames.get(name);
+    if (!renamed) throw new Error(`unresolved $ref "${node.$ref}"`);
+    node.$ref = `#/components/schemas/${renamed}`;
+  });
+}
+
+export function buildSchemas(): StateSchemas {
+  const raw = buildRawSchema() as AnySchema;
+  const { definitions = {}, $schema: _schema, ...root } = raw;
+
+  stripInternal(root);
+  Object.values(definitions as Record<string, AnySchema>).forEach(stripInternal);
+
+  const defs = reachableDefs(root, definitions);
+
+  const finalNames = new Map<string, string>();
+  for (const name of Object.keys(defs)) {
+    const renamed = RENAME[name] ?? name;
+    if (!VALID_NAME.test(renamed)) {
+      throw new Error(`invalid component name "${renamed}", rename the type in evcc.ts`);
+    }
+    if ([...finalNames.values()].includes(renamed)) {
+      throw new Error(`duplicate component name "${renamed}" after renaming`);
+    }
+    finalNames.set(name, renamed);
+  }
+
+  rewriteRefs(root, finalNames);
+  Object.values(defs).forEach((schema) => rewriteRefs(schema, finalNames));
+
+  normalizeNullables(root);
+  Object.values(defs).forEach(normalizeNullables);
+
+  const result: Record<string, Schema> = { [ROOT]: root };
+  for (const [name, schema] of Object.entries(defs)
+    .map(([name, schema]) => [finalNames.get(name)!, schema] as const)
+    .sort(([a], [b]) => a.localeCompare(b))) {
+    result[name] = schema;
+  }
+
+  const serialized = JSON.stringify(result);
+  if (serialized.includes("#/definitions/")) {
+    throw new Error("unrewritten $ref to #/definitions/ left in output");
+  }
+
+  const ordered = Object.fromEntries(
+    Object.entries(result).map(([name, schema]) => [name, orderKeys(schema)])
+  );
+
+  return { rootName: ROOT, defs: ordered };
+}
+
+const KEY_ORDER = [
+  "$ref",
+  "description",
+  "type",
+  "nullable",
+  "enum",
+  "format",
+  "example",
+  "items",
+  "properties",
+  "required",
+  "additionalProperties",
+  "anyOf",
+  "allOf",
+];
+
+// stable key order for readable yaml diffs, property order itself is preserved
+function orderKeys(node: any): any {
+  if (Array.isArray(node)) return node.map(orderKeys);
+  if (!node || typeof node !== "object") return node;
+  const keys = Object.keys(node).sort((a, b) => {
+    const ia = KEY_ORDER.indexOf(a);
+    const ib = KEY_ORDER.indexOf(b);
+    return (ia === -1 ? KEY_ORDER.length : ia) - (ib === -1 ? KEY_ORDER.length : ib);
+  });
+  return Object.fromEntries(
+    keys.map((key) => [
+      key,
+      key === "properties" ? mapValues(node[key], orderKeys) : orderKeys(node[key]),
+    ])
+  );
+}
+
+function mapValues(obj: Record<string, any>, fn: (v: any) => any): Record<string, any> {
+  return Object.fromEntries(Object.entries(obj).map(([k, v]) => [k, fn(v)]));
+}

scripts/state-schema/validate.ts

@@ -0,0 +1,114 @@
+import { readFileSync } from "node:fs";
+import { parse } from "yaml";
+import { Ajv2020 } from "ajv/dist/2020.js";
+import addFormats from "ajv-formats";
+
+const STATE_SCHEMAS_PATH = "server/openapi.state.yaml";
+
+// intentionally undocumented experimental structures
+const IGNORE = new Set(["$.evopt", "$.evopt-batteries"]);
+
+type AnySchema = any;
+
+function resolve(schema: AnySchema, schemas: Record<string, AnySchema>): AnySchema {
+  if (typeof schema?.$ref === "string") {
+    return resolve(schemas[schema.$ref.split("/").pop()!], schemas);
+  }
+  return schema ?? {};
+}
+
+// report payload keys that have no schema property, additionalProperties:true hides them from ajv
+function coverage(
+  schema: AnySchema,
+  data: any,
+  path: string,
+  schemas: Record<string, AnySchema>,
+  report: Set<string>
+): void {
+  const s = resolve(schema, schemas);
+  if (s.anyOf) {
+    if (data === null) return;
+    const branch = s.anyOf.find((b: AnySchema) => resolve(b, schemas).type !== "null");
+    if (branch) coverage(branch, data, path, schemas, report);
+    return;
+  }
+  if (Array.isArray(data)) {
+    if (s.items) data.forEach((item) => coverage(s.items, item, `${path}[]`, schemas, report));
+    return;
+  }
+  if (data !== null && typeof data === "object") {
+    if (!s.properties) return;
+    for (const key of Object.keys(data)) {
+      if (s.properties[key]) {
+        coverage(s.properties[key], data[key], `${path}.${key}`, schemas, report);
+      } else if (!IGNORE.has(`${path}.${key}`)) {
+        report.add(`${path}.${key}`);
+      }
+    }
+  }
+}
+
+// openapi uses `nullable: true`, json schema expects explicit null in the type
+function expandNullable(node: any): void {
+  if (Array.isArray(node)) {
+    node.forEach(expandNullable);
+  } else if (node && typeof node === "object") {
+    if (node.nullable === true) {
+      delete node.nullable;
+      if (typeof node.type === "string") {
+        node.type = [node.type, "null"];
+      } else if (Array.isArray(node.allOf)) {
+        node.anyOf = [...node.allOf, { type: "null" }];
+        delete node.allOf;
+      }
+    }
+    Object.values(node).forEach(expandNullable);
+  }
+}
+
+// validate a /api/state payload against the State schema in server/openapi.state.yaml
+export function validateState(payload: any): { errors: string[]; undocumented: string[] } {
+  const doc = parse(readFileSync(STATE_SCHEMAS_PATH, "utf8"));
+  const schemas: Record<string, AnySchema> = doc.components.schemas;
+  expandNullable(schemas);
+
+  const ajv = new Ajv2020({ strict: false, allErrors: true });
+  addFormats(ajv);
+  const validateFn = ajv.compile({
+    $ref: "#/components/schemas/State",
+    components: { schemas },
+  });
+
+  const errors = validateFn(payload)
+    ? []
+    : (validateFn.errors ?? []).map((err) => `${err.instancePath || "/"} ${err.message}`);
+
+  const undocumented = new Set<string>();
+  coverage(schemas["State"], payload, "$", schemas, undocumented);
+
+  return { errors, undocumented: [...undocumented].sort() };
+}
+
+export async function validate(source: string): Promise<boolean> {
+  let payload: any;
+  if (source.startsWith("http")) {
+    const res = await fetch(`${source.replace(/\/$/, "")}/api/state`);
+    payload = await res.json();
+  } else {
+    payload = JSON.parse(readFileSync(source, "utf8"));
+  }
+
+  const { errors, undocumented } = validateState(payload);
+
+  if (errors.length > 0) {
+    console.error(`${source}: schema violations`);
+    for (const err of errors.slice(0, 30)) console.error(`  ${err}`);
+  }
+  if (undocumented.length > 0) {
+    console.log(`${source}: ${undocumented.length} undocumented keys`);
+    for (const key of undocumented) console.log(`  ${key}`);
+  }
+  if (errors.length === 0 && undocumented.length === 0) console.log(`${source}: ok`);
+
+  return errors.length === 0;
+}