diff --git a/core/site.go b/core/site.go index 6f0191616..d0a7e36b2 100644 --- a/core/site.go +++ b/core/site.go @@ -35,6 +35,7 @@ import ( "github.com/evcc-io/evcc/util/config" "github.com/evcc-io/evcc/util/modbus" "github.com/evcc-io/evcc/util/telemetry" + "github.com/jinzhu/now" "github.com/samber/lo" "github.com/smallnest/chanx" "golang.org/x/sync/errgroup" @@ -120,6 +121,8 @@ type Site struct { optimizerMu sync.Mutex // guards optimizer runs optimizerUpdated time.Time // last optimizer run, guarded by optimizerMu + + solarScaleCached func() (float64, error) // util.Cached wrapper around querySolarScale } // MetersConfig contains the site's meter configuration @@ -354,6 +357,15 @@ func NewSite() *Site { collectors: make(map[string]*metrics.Collector), } + // the result only depends on completed days, so it cannot change within a day + site.solarScaleCached = util.Cached(func() (float64, error) { + scale, err := site.querySolarScale(now.BeginningOfDay()) + if err != nil { + site.log.ERROR.Printf("solar scale percentile: %v, falling back to unadjusted forecast", err) + } + return scale, err + }, 24*time.Hour) + return site } diff --git a/core/site_tariffs.go b/core/site_tariffs.go index b62c99bc7..e45e806d1 100644 --- a/core/site_tariffs.go +++ b/core/site_tariffs.go @@ -3,6 +3,7 @@ package core import ( "encoding/json" "math" + "slices" "time" "github.com/evcc-io/evcc/api" @@ -26,7 +27,7 @@ func (s forecastSeries) MarshalBytes() ([]byte, error) { } type solarDetails struct { - Scale float64 `json:"scale"` // scale factor yield/forecasted today, 1 if unscaled + Scale float64 `json:"scale"` // trailing percentile solar scale factor, 1 if unscaled Today dailyDetails `json:"today"` // tomorrow Tomorrow dailyDetails `json:"tomorrow"` // tomorrow DayAfterTomorrow dailyDetails `json:"dayAfterTomorrow"` // day after tomorrow @@ -239,8 +240,8 @@ func (site *Site) solarDetails(solar api.Rates) solarDetails { return res } -// effectiveSolarScale returns the solar forecast scale if forecast adjustment -// is enabled, 1 otherwise. +// effectiveSolarScale returns the solar forecast scale used to adjust the +// optimizer's solar input if forecast adjustment is enabled, 1 otherwise. func (site *Site) effectiveSolarScale() float64 { if !site.GetSolarAdjusted() { return 1 @@ -248,39 +249,87 @@ func (site *Site) effectiveSolarScale() float64 { return site.solarScale() } -// solarScale returns the ratio of produced solar energy to forecasted solar -// energy for the current day, queried from the metrics database. Used to -// adjust forecasts when PV is consistently under-/over-producing relative -// to the forecast. Returns 1.0 when not enough data is available to make -// the ratio meaningful. +const ( + solarScaleWindow = 30 // trailing window of days to consider + solarScaleMinSamples = 14 // minimum daily ratios before applying a scale + solarScalePercentile = 0.5 // percentile of the daily ratio distribution to use + solarScaleMinEnergy = 0.5 // kWh, skip days where either side is too small for a meaningful ratio +) + +// solarScale computes a scale factor for the solar forecast by sorting the daily +// produced/forecasted solar ratio over a trailing window of completed days and +// picking the value at a configured percentile (window: solarScaleWindow, +// percentile: solarScalePercentile). This captures the installation's systematic +// bias (soiling, shading, model error) instead of a single day's weather noise. +// The current (partial) day is excluded; returns 1 when there is not enough history. +// +// Depends only on completed days, so it's cached instead of recomputed per run. +// +// The result only depends on completed days, so it cannot change within a day. It is +// cached accordingly instead of being recomputed on every optimizer run. func (site *Site) solarScale() float64 { - series, err := metrics.QueryEnergy(now.BeginningOfDay(), time.Now(), "day", true) + scale, err := site.solarScaleCached() if err != nil { - site.log.ERROR.Printf("solar forecast scale: %v", err) return 1 } + return scale +} - var pv, fcst float64 +// querySolarScale does the actual metrics query and percentile calculation +// for solarScale, given the current beginning-of-day boundary. +func (site *Site) querySolarScale(bod time.Time) (float64, error) { + from := bod.AddDate(0, 0, -solarScaleWindow) + series, err := metrics.QueryEnergy(from, time.Now(), "day", true) + if err != nil { + return 0, err + } + + pv := make(map[string]float64, solarScaleWindow) + fcst := make(map[string]float64, solarScaleWindow) for _, s := range series { - if len(s.Data) == 0 { - continue - } + var m map[string]float64 switch s.Group { case metrics.PV: - pv = s.Data[0].Energy + m = pv case metrics.Forecast: - fcst = s.Data[0].Energy + m = fcst + default: + continue + } + for _, d := range s.Data { + m[d.Start.Format("2006-01-02")] = d.Energy } } - const minEnergy = 0.5 // kWh - if fcst <= 0 || pv <= minEnergy { - return 1 + today := bod.Format("2006-01-02") + ratios := make([]float64, 0, len(fcst)) + for day, f := range fcst { + // skip today (partial) and dark days where the ratio is noise. The threshold + // applies to production as well: a near-zero yield against a healthy forecast + // is a fault (snow, soiling, inverter or metering outage), not a bias that + // should be projected onto the next solarScaleWindow days. + if p := pv[day]; day != today && f > solarScaleMinEnergy && p > solarScaleMinEnergy { + ratios = append(ratios, p/f) + } } - scale := pv / fcst - site.log.DEBUG.Printf("solar forecast: produced %.3fkWh, forecasted %.3fkWh, scale %.3f", pv, fcst, scale) - return scale + scale, ok := percentileOf(ratios, solarScalePercentile, solarScaleMinSamples) + if !ok { + return 1, nil + } + site.log.DEBUG.Printf("solar scale P%.0f over %d days = %.3f", solarScalePercentile*100, len(ratios), scale) + return scale, nil +} + +// percentileOf returns the p-th percentile (0..1) of values by nearest-rank on the +// sorted series, or false when fewer than minSamples are present. +func percentileOf(values []float64, p float64, minSamples int) (float64, bool) { + if len(values) < minSamples { + return 0, false + } + s := slices.Clone(values) + slices.Sort(s) + return s[int(p*float64(len(s)-1))], true } func (site *Site) isDynamicTariff(usage api.TariffUsage) bool { diff --git a/core/site_tariffs_test.go b/core/site_tariffs_test.go index c8dd322ce..73d6d778a 100644 --- a/core/site_tariffs_test.go +++ b/core/site_tariffs_test.go @@ -92,3 +92,49 @@ func TestTimeseriesMarshal(t *testing.T) { }) } } + +func TestPercentileOf(t *testing.T) { + // n values of v + fill := func(n int, v float64) []float64 { + s := make([]float64, n) + for i := range s { + s[i] = v + } + return s + } + + t.Run("too few samples returns false", func(t *testing.T) { + _, ok := percentileOf(nil, 0.5, solarScaleMinSamples) + assert.False(t, ok) + + _, ok = percentileOf(fill(solarScaleMinSamples-1, 0.9), 0.5, solarScaleMinSamples) + assert.False(t, ok) + }) + + t.Run("stable cluster", func(t *testing.T) { + v, ok := percentileOf(fill(20, 0.9), 0.5, solarScaleMinSamples) + assert.True(t, ok) + assert.InDelta(t, 0.9, v, 0.001) + }) + + // P50 rejects outlier days for free: a broken forecast feed (recent ratio + // ~2.3) and a metering outage (ratio ~0.16) do not move the result as + // long as they stay a minority of the window. + t.Run("outlier days do not move P50", func(t *testing.T) { + ratios := fill(20, 0.9) // healthy installation bias + ratios = append(ratios, fill(4, 2.3)...) // broken forecast feed + ratios = append(ratios, fill(8, 0.16)...) // metering outage + + v, ok := percentileOf(ratios, 0.5, solarScaleMinSamples) + assert.True(t, ok) + assert.InDelta(t, 0.9, v, 0.001) + }) + + t.Run("higher percentile shifts toward the upper tail", func(t *testing.T) { + ratios := append(fill(15, 0.8), fill(15, 1.2)...) + + p50, _ := percentileOf(ratios, 0.5, solarScaleMinSamples) + p90, _ := percentileOf(ratios, 0.9, solarScaleMinSamples) + assert.Less(t, p50, p90) + }) +} diff --git a/server/http_db_metrics_handler.go b/server/http_db_metrics_handler.go index 05d1423c4..f606e549b 100644 --- a/server/http_db_metrics_handler.go +++ b/server/http_db_metrics_handler.go @@ -7,6 +7,7 @@ import ( "github.com/evcc-io/evcc/core/metrics" "github.com/evcc-io/evcc/server/db" + "github.com/evcc-io/evcc/util" ) // deleteResult reports the number of affected rows @@ -58,6 +59,9 @@ func deleteEnergyHandler(w http.ResponseWriter, r *http.Request) { return } + // invalidate cached values (e.g. the solar scale) derived from the deleted history + util.ResetCached() + jsonWrite(w, deleteResult{rows}) }