From 5ab5cfc2efd2506c5202f032dc00c9bb43cbe46c Mon Sep 17 00:00:00 2001 From: andig Date: Sun, 9 Aug 2026 16:30:34 +0200 Subject: [PATCH] Add admin api for deleting invalid metrics (#32666) --- core/metrics/db_delete_test.go | 104 +++++++++++++++++ core/metrics/db_history.go | 53 +++++++-- core/metrics/tariffs.go | 48 ++++++++ server/http.go | 8 +- server/http_db_metrics_handler.go | 88 +++++++++++++++ server/mcp/openapi.json | 181 ++++++++++++++++++++++++++++++ server/openapi.yaml | 119 ++++++++++++++++++++ 7 files changed, 589 insertions(+), 12 deletions(-) create mode 100644 core/metrics/db_delete_test.go create mode 100644 server/http_db_metrics_handler.go diff --git a/core/metrics/db_delete_test.go b/core/metrics/db_delete_test.go new file mode 100644 index 000000000..a68ee8d25 --- /dev/null +++ b/core/metrics/db_delete_test.go @@ -0,0 +1,104 @@ +package metrics + +import ( + "testing" + "time" + + "github.com/evcc-io/evcc/server/db" + "github.com/stretchr/testify/require" +) + +func TestDeleteEnergy(t *testing.T) { + require.NoError(t, db.NewInstance("sqlite", ":memory:")) + require.NoError(t, SetupSchema()) + + ePv := entity{Id: 2, Name: "pv1", Group: PV} + require.NoError(t, db.Instance.Create(&ePv).Error) + eFc := entity{Id: 3, Name: Forecast, Group: Forecast} + require.NoError(t, db.Instance.Create(&eFc).Error) + + loc := time.Now().Location() + base := time.Date(2026, 4, 15, 16, 0, 0, 0, loc) + for i := range 4 { + ts := base.Add(time.Duration(i) * 15 * time.Minute) + require.NoError(t, persist(ePv, ts, 1, 0, nil, false)) + require.NoError(t, persist(eFc, ts, 2, 0, nil, false)) + } + + count := func() int64 { + var n int64 + require.NoError(t, db.Instance.Model(new(meter)).Count(&n).Error) + return n + } + require.Equal(t, int64(8), count()) + + // both bounds are required + _, err := DeleteEnergy(time.Time{}, base, EnergyFilter{}) + require.Error(t, err) + + // filter narrows to the matching entity, range is half-open + rows, err := DeleteEnergy(base.UTC(), base.Add(30*time.Minute).UTC(), EnergyFilter{Group: Forecast}) + require.NoError(t, err) + require.Equal(t, int64(2), rows) + require.Equal(t, int64(6), count()) + + // pv untouched + res, err := QueryEnergy(base.Add(-time.Hour).UTC(), base.Add(time.Hour).UTC(), "hour", false, EnergyFilter{Group: PV}) + require.NoError(t, err) + require.Len(t, res, 1) + require.Equal(t, 4.0, res[0].Data[0].Energy) + + // empty filter deletes across entities + rows, err = DeleteEnergy(base.UTC(), base.Add(time.Hour).UTC(), EnergyFilter{}) + require.NoError(t, err) + require.Equal(t, int64(6), rows) + require.Zero(t, count()) +} + +func TestDeleteTariffs(t *testing.T) { + require.NoError(t, db.NewInstance("sqlite", ":memory:")) + require.NoError(t, db.Instance.AutoMigrate(new(tariffValue))) + + base := time.Date(2026, 4, 15, 16, 0, 0, 0, time.UTC) + grid, co2 := 0.3, 250.0 + for i := range 4 { + require.NoError(t, PersistTariffs(base.Add(time.Duration(i)*15*time.Minute), &grid, nil, &co2, nil)) + } + + count := func() int64 { + var n int64 + require.NoError(t, db.Instance.Model(new(tariffValue)).Count(&n).Error) + return n + } + + _, err := DeleteTariffs(base, time.Time{}, "") + require.Error(t, err) + + // unknown usage never reaches the column interpolation + _, err = DeleteTariffs(base, base.Add(time.Hour), "unknown") + require.ErrorIs(t, err, ErrInvalidUsage) + require.Equal(t, int64(4), count()) + + // clearing one usage keeps the row as long as another value remains + rows, err := DeleteTariffs(base, base.Add(30*time.Minute), "grid") + require.NoError(t, err) + require.Equal(t, int64(2), rows) + require.Equal(t, int64(4), count()) + + var res tariffValue + require.NoError(t, db.Instance.Where("ts = ?", base.Unix()).First(&res).Error) + require.Nil(t, res.Grid) + require.InDelta(t, 250, *res.Co2, 0.001) + + // clearing the last usage drops the now empty rows + rows, err = DeleteTariffs(base, base.Add(30*time.Minute), "co2") + require.NoError(t, err) + require.Equal(t, int64(2), rows) + require.Equal(t, int64(2), count()) + + // no usage drops the whole row + rows, err = DeleteTariffs(base, base.Add(time.Hour), "") + require.NoError(t, err) + require.Equal(t, int64(2), rows) + require.Zero(t, count()) +} diff --git a/core/metrics/db_history.go b/core/metrics/db_history.go index 8aa93cd6f..b410c5287 100644 --- a/core/metrics/db_history.go +++ b/core/metrics/db_history.go @@ -10,6 +10,7 @@ import ( "github.com/evcc-io/evcc/server/db" "github.com/evcc-io/evcc/util/export" + "gorm.io/gorm" ) // Slot represents an aggregated energy time slot @@ -62,6 +63,47 @@ type EnergyFilter struct { Title string } +// entityQuery returns a subquery selecting the ids of the matching entities, +// nil for an empty filter. +func entityQuery(f EnergyFilter) *gorm.DB { + if f.Group == "" && f.Name == "" && f.Title == "" { + return nil + } + + tx := db.Instance.Model(new(entity)).Select("id") + if f.Group != "" { + tx = tx.Where(`"group" = ?`, f.Group) + } + if f.Name != "" { + tx = tx.Where("name = ?", f.Name) + } + if f.Title != "" { + tx = tx.Where("title = ?", f.Title) + } + + return tx +} + +// DeleteEnergy removes the slots in [from,to), narrowed to the matching +// entities. Both bounds are required, a full wipe is /api/db/reset. +func DeleteEnergy(from, to time.Time, filter ...EnergyFilter) (int64, error) { + if from.IsZero() || to.IsZero() { + return 0, errors.New("missing from/to") + } + + tx := db.Instance.Where("ts >= ? AND ts < ?", from.Unix(), to.Unix()) + + if len(filter) > 0 { + if sub := entityQuery(filter[0]); sub != nil { + tx = tx.Where("meter IN (?)", sub) + } + } + + res := tx.Delete(new(meter)) + + return res.RowsAffected, res.Error +} + // QueryEnergy returns aggregated energy data, per title or per group. func QueryEnergy(from, to time.Time, aggregate string, grouped bool, filter ...EnergyFilter) ([]Series, error) { addDuration := aggregateDurations[aggregate] @@ -114,15 +156,8 @@ func QueryEnergy(from, to time.Time, aggregate string, grouped bool, filter ...E } if len(filter) > 0 { - f := filter[0] - if f.Group != "" { - tx = tx.Where(`e."group" = ?`, f.Group) - } - if f.Name != "" { - tx = tx.Where("e.name = ?", f.Name) - } - if f.Title != "" { - tx = tx.Where("e.title = ?", f.Title) + if sub := entityQuery(filter[0]); sub != nil { + tx = tx.Where("m.meter IN (?)", sub) } } diff --git a/core/metrics/tariffs.go b/core/metrics/tariffs.go index 66a04b2fb..ecd3ecfa0 100644 --- a/core/metrics/tariffs.go +++ b/core/metrics/tariffs.go @@ -1,6 +1,10 @@ package metrics import ( + "errors" + "fmt" + "slices" + "strings" "time" "github.com/evcc-io/evcc/server/db" @@ -26,6 +30,50 @@ func init() { }) } +// ErrInvalidUsage is returned for an unknown tariff usage +var ErrInvalidUsage = errors.New("invalid usage") + +// tariffUsages are the deletable usages, named after their table column +var tariffUsages = []string{"grid", "feedin", "co2", "temperature"} + +// DeleteTariffs removes the persisted values in [from,to). An empty usage drops +// the entire row, otherwise only that usage is cleared. Both bounds are +// required, a full wipe is /api/db/reset. The count is the number of affected +// rows; rows dropped by the cleanup are a subset of the cleared ones. +func DeleteTariffs(from, to time.Time, usage string) (int64, error) { + if from.IsZero() || to.IsZero() { + return 0, errors.New("missing from/to") + } + + inRange := func() *gorm.DB { + return db.Instance.Where("ts >= ? AND ts < ?", from.Unix(), to.Unix()) + } + + if usage == "" { + res := inRange().Delete(new(tariffValue)) + return res.RowsAffected, res.Error + } + + // guards the column interpolated below + if !slices.Contains(tariffUsages, usage) { + return 0, fmt.Errorf("%w: %s (valid: %s)", ErrInvalidUsage, usage, strings.Join(tariffUsages, ", ")) + } + + res := inRange().Model(new(tariffValue)). + Where(usage+" IS NOT NULL"). + Update(usage, gorm.Expr("NULL")) + if res.Error != nil { + return 0, res.Error + } + + // drop the rows that no longer hold any value + err := inRange(). + Where("grid IS NULL AND feedin IS NULL AND co2 IS NULL AND temperature IS NULL"). + Delete(new(tariffValue)).Error + + return res.RowsAffected, err +} + // PersistTariffs stores the tariff values at the given 15min boundary, nil values omitted func PersistTariffs(ts time.Time, grid, feedin, co2, temperature *float64) error { if grid == nil && feedin == nil && co2 == nil && temperature == nil { diff --git a/server/http.go b/server/http.go index b1aa3f0de..7b91dabc0 100644 --- a/server/http.go +++ b/server/http.go @@ -438,9 +438,11 @@ func (s *HTTPd) RegisterSystemHandler(site *core.Site, pub publisher, cache *uti api.Use(ensureDbAuth(auth)) routes := map[string]route{ - "backup": {"GET", "/backup", getBackup()}, - "restore": {"POST", "/restore", restoreDatabase(shutdown)}, - "reset": {"POST", "/reset", resetDatabase(shutdown)}, + "backup": {"GET", "/backup", getBackup()}, + "restore": {"POST", "/restore", restoreDatabase(shutdown)}, + "reset": {"POST", "/reset", resetDatabase(shutdown)}, + "deleteenergy": {"DELETE", "/metrics/energy", deleteEnergyHandler}, + "deletetariffs": {"DELETE", "/metrics/tariffs", deleteTariffsHandler}, } for _, r := range routes { diff --git a/server/http_db_metrics_handler.go b/server/http_db_metrics_handler.go new file mode 100644 index 000000000..05d1423c4 --- /dev/null +++ b/server/http_db_metrics_handler.go @@ -0,0 +1,88 @@ +package server + +import ( + "errors" + "net/http" + "time" + + "github.com/evcc-io/evcc/core/metrics" + "github.com/evcc-io/evcc/server/db" +) + +// deleteResult reports the number of affected rows +type deleteResult struct { + Deleted int64 `json:"deleted"` +} + +// timeRange parses the mandatory from/to query parameters +func timeRange(r *http.Request) (time.Time, time.Time, error) { + q := r.URL.Query() + + from, err := time.Parse(time.RFC3339, q.Get("from")) + if err != nil { + return time.Time{}, time.Time{}, errors.New("invalid 'from' parameter") + } + + to, err := time.Parse(time.RFC3339, q.Get("to")) + if err != nil { + return time.Time{}, time.Time{}, errors.New("invalid 'to' parameter") + } + + return from, to, nil +} + +// deleteEnergyHandler removes energy metrics. Parameters match +// /api/history/energy so the same request can be previewed before deleting. +func deleteEnergyHandler(w http.ResponseWriter, r *http.Request) { + if db.Instance == nil { + jsonError(w, http.StatusBadRequest, errors.New("database offline")) + return + } + + from, to, err := timeRange(r) + if err != nil { + jsonError(w, http.StatusBadRequest, err) + return + } + + q := r.URL.Query() + filter := metrics.EnergyFilter{ + Group: q.Get("group"), + Name: q.Get("name"), + Title: q.Get("title"), + } + + rows, err := metrics.DeleteEnergy(from, to, filter) + if err != nil { + jsonError(w, http.StatusInternalServerError, err) + return + } + + jsonWrite(w, deleteResult{rows}) +} + +// deleteTariffsHandler removes persisted tariff values +func deleteTariffsHandler(w http.ResponseWriter, r *http.Request) { + if db.Instance == nil { + jsonError(w, http.StatusBadRequest, errors.New("database offline")) + return + } + + from, to, err := timeRange(r) + if err != nil { + jsonError(w, http.StatusBadRequest, err) + return + } + + rows, err := metrics.DeleteTariffs(from, to, r.URL.Query().Get("usage")) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, metrics.ErrInvalidUsage) { + status = http.StatusBadRequest + } + jsonError(w, status, err) + return + } + + jsonWrite(w, deleteResult{rows}) +} diff --git a/server/mcp/openapi.json b/server/mcp/openapi.json index ee7fa6457..6c0f36efd 100644 --- a/server/mcp/openapi.json +++ b/server/mcp/openapi.json @@ -2244,6 +2244,171 @@ } } }, + "/db/metrics/energy": { + "delete": { + "operationId": "deleteEnergyMetrics", + "summary": "Delete energy metrics", + "description": "Deletes recorded energy slots in the given time range, optionally narrowed by entity. Filter parameters match /history/energy, so the same request can be previewed there before deleting. Session users must supply the admin password in the X-Admin-Password header. API key holders via Bearer token are exempt.", + "tags": [ + "db" + ], + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/adminPassword" + }, + { + "name": "from", + "in": "query", + "required": true, + "description": "Start time, inclusive (RFC3339)", + "schema": { + "type": "string", + "format": "date-time", + "example": "2026-07-01T00:00:00Z" + } + }, + { + "name": "to", + "in": "query", + "required": true, + "description": "End time, exclusive (RFC3339)", + "schema": { + "type": "string", + "format": "date-time", + "example": "2026-07-02T00:00:00Z" + } + }, + { + "name": "group", + "in": "query", + "description": "Filter by entity group", + "schema": { + "type": "string", + "example": "forecast", + "enum": [ + "forecast", + "battery", + "grid", + "pv", + "home", + "loadpoint", + "meter", + "consumer", + "temperature" + ] + } + }, + { + "name": "name", + "in": "query", + "description": "Filter by entity name", + "schema": { + "type": "string", + "example": "db:8" + } + }, + { + "name": "title", + "in": "query", + "description": "Filter by entity title", + "schema": { + "type": "string", + "example": "Battery" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Deleted" + }, + "400": { + "description": "Invalid parameters or database offline" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + } + } + } + }, + "/db/metrics/tariffs": { + "delete": { + "operationId": "deleteTariffMetrics", + "summary": "Delete tariff metrics", + "description": "Deletes recorded tariff values in the given time range. Without usage the entire record is removed, otherwise only that usage is cleared and records left without any value are removed. Session users must supply the admin password in the X-Admin-Password header. API key holders via Bearer token are exempt.", + "tags": [ + "db" + ], + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/adminPassword" + }, + { + "name": "from", + "in": "query", + "required": true, + "description": "Start time, inclusive (RFC3339)", + "schema": { + "type": "string", + "format": "date-time", + "example": "2026-07-01T00:00:00Z" + } + }, + { + "name": "to", + "in": "query", + "required": true, + "description": "End time, exclusive (RFC3339)", + "schema": { + "type": "string", + "format": "date-time", + "example": "2026-07-02T00:00:00Z" + } + }, + { + "name": "usage", + "in": "query", + "description": "Limit deletion to a single tariff usage", + "schema": { + "type": "string", + "example": "co2", + "enum": [ + "grid", + "feedin", + "co2", + "temperature" + ] + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Deleted" + }, + "400": { + "description": "Invalid parameters or database offline" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + } + } + } + }, "/history/energy": { "get": { "operationId": "getEnergyHistory", @@ -5296,6 +5461,22 @@ } }, "responses": { + "Deleted": { + "description": "Success - number of deleted records", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "deleted": { + "type": "integer", + "example": 96 + } + } + } + } + } + }, "NanoSecondsResult": { "description": "Success - Number result - Unit is **nanoseconds**", "content": { diff --git a/server/openapi.yaml b/server/openapi.yaml index d1f16eacc..3bafd8e6c 100644 --- a/server/openapi.yaml +++ b/server/openapi.yaml @@ -1353,6 +1353,115 @@ paths: description: Reset successful, instance is restarting "401": $ref: "#/components/responses/Unauthorized" + /db/metrics/energy: + delete: + operationId: deleteEnergyMetrics + summary: Delete energy metrics + description: "Deletes recorded energy slots in the given time range, optionally narrowed by entity. Filter parameters match /history/energy, so the same request can be previewed there before deleting. Session users must supply the admin password in the X-Admin-Password header. API key holders via Bearer token are exempt." + tags: + - db + security: + - cookieAuth: [] + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/adminPassword" + - name: from + in: query + required: true + description: Start time, inclusive (RFC3339) + schema: + type: string + format: date-time + example: "2026-07-01T00:00:00Z" + - name: to + in: query + required: true + description: End time, exclusive (RFC3339) + schema: + type: string + format: date-time + example: "2026-07-02T00:00:00Z" + - name: group + in: query + description: Filter by entity group + schema: + type: string + example: forecast + enum: + - forecast + - battery + - grid + - pv + - home + - loadpoint + - meter + - consumer + - temperature + - name: name + in: query + description: Filter by entity name + schema: + type: string + example: db:8 + - name: title + in: query + description: Filter by entity title + schema: + type: string + example: Battery + responses: + "200": + $ref: "#/components/responses/Deleted" + "400": + description: Invalid parameters or database offline + "401": + $ref: "#/components/responses/Unauthorized" + /db/metrics/tariffs: + delete: + operationId: deleteTariffMetrics + summary: Delete tariff metrics + description: "Deletes recorded tariff values in the given time range. Without usage the entire record is removed, otherwise only that usage is cleared and records left without any value are removed. Session users must supply the admin password in the X-Admin-Password header. API key holders via Bearer token are exempt." + tags: + - db + security: + - cookieAuth: [] + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/adminPassword" + - name: from + in: query + required: true + description: Start time, inclusive (RFC3339) + schema: + type: string + format: date-time + example: "2026-07-01T00:00:00Z" + - name: to + in: query + required: true + description: End time, exclusive (RFC3339) + schema: + type: string + format: date-time + example: "2026-07-02T00:00:00Z" + - name: usage + in: query + description: Limit deletion to a single tariff usage + schema: + type: string + example: co2 + enum: + - grid + - feedin + - co2 + - temperature + responses: + "200": + $ref: "#/components/responses/Deleted" + "400": + description: Invalid parameters or database offline + "401": + $ref: "#/components/responses/Unauthorized" /history/energy: get: operationId: getEnergyHistory @@ -1816,6 +1925,16 @@ components: - "1" - "0" responses: + Deleted: + description: Success - number of deleted records + content: + application/json: + schema: + type: object + properties: + deleted: + type: integer + example: 96 NanoSecondsResult: description: Success - Number result - Unit is **nanoseconds** content: