Site: feed the optimizer a trailing percentile solar scale (#32849)

core/site.go

@@ -35,6 +35,7 @@ import (
 	"github.com/evcc-io/evcc/util/config"
 	"github.com/evcc-io/evcc/util/modbus"
 	"github.com/evcc-io/evcc/util/telemetry"
+	"github.com/jinzhu/now"
 	"github.com/samber/lo"
 	"github.com/smallnest/chanx"
 	"golang.org/x/sync/errgroup"
@@ -120,6 +121,8 @@ type Site struct {
 
 	optimizerMu      sync.Mutex // guards optimizer runs
 	optimizerUpdated time.Time  // last optimizer run, guarded by optimizerMu
+
+	solarScaleCached func() (float64, error) // util.Cached wrapper around querySolarScale
 }
 
 // MetersConfig contains the site's meter configuration
@@ -354,6 +357,15 @@ func NewSite() *Site {
 		collectors: make(map[string]*metrics.Collector),
 	}
 
+	// the result only depends on completed days, so it cannot change within a day
+	site.solarScaleCached = util.Cached(func() (float64, error) {
+		scale, err := site.querySolarScale(now.BeginningOfDay())
+		if err != nil {
+			site.log.ERROR.Printf("solar scale percentile: %v, falling back to unadjusted forecast", err)
+		}
+		return scale, err
+	}, 24*time.Hour)
+
 	return site
 }
 

core/site_tariffs.go

@@ -3,6 +3,7 @@ package core
 import (
 	"encoding/json"
 	"math"
+	"slices"
 	"time"
 
 	"github.com/evcc-io/evcc/api"
@@ -26,7 +27,7 @@ func (s forecastSeries) MarshalBytes() ([]byte, error) {
 }
 
 type solarDetails struct {
-	Scale            float64      `json:"scale"`                // scale factor yield/forecasted today, 1 if unscaled
+	Scale            float64      `json:"scale"`                // trailing percentile solar scale factor, 1 if unscaled
 	Today            dailyDetails `json:"today"`                // tomorrow
 	Tomorrow         dailyDetails `json:"tomorrow"`             // tomorrow
 	DayAfterTomorrow dailyDetails `json:"dayAfterTomorrow"`     // day after tomorrow
@@ -239,8 +240,8 @@ func (site *Site) solarDetails(solar api.Rates) solarDetails {
 	return res
 }
 
-// effectiveSolarScale returns the solar forecast scale if forecast adjustment
-// is enabled, 1 otherwise.
+// effectiveSolarScale returns the solar forecast scale used to adjust the
+// optimizer's solar input if forecast adjustment is enabled, 1 otherwise.
 func (site *Site) effectiveSolarScale() float64 {
 	if !site.GetSolarAdjusted() {
 		return 1
@@ -248,39 +249,87 @@ func (site *Site) effectiveSolarScale() float64 {
 	return site.solarScale()
 }
 
-// solarScale returns the ratio of produced solar energy to forecasted solar
-// energy for the current day, queried from the metrics database. Used to
-// adjust forecasts when PV is consistently under-/over-producing relative
-// to the forecast. Returns 1.0 when not enough data is available to make
-// the ratio meaningful.
+const (
+	solarScaleWindow     = 30  // trailing window of days to consider
+	solarScaleMinSamples = 14  // minimum daily ratios before applying a scale
+	solarScalePercentile = 0.5 // percentile of the daily ratio distribution to use
+	solarScaleMinEnergy  = 0.5 // kWh, skip days where either side is too small for a meaningful ratio
+)
+
+// solarScale computes a scale factor for the solar forecast by sorting the daily
+// produced/forecasted solar ratio over a trailing window of completed days and
+// picking the value at a configured percentile (window: solarScaleWindow,
+// percentile: solarScalePercentile). This captures the installation's systematic
+// bias (soiling, shading, model error) instead of a single day's weather noise.
+// The current (partial) day is excluded; returns 1 when there is not enough history.
+//
+// Depends only on completed days, so it's cached instead of recomputed per run.
+//
+// The result only depends on completed days, so it cannot change within a day. It is
+// cached accordingly instead of being recomputed on every optimizer run.
 func (site *Site) solarScale() float64 {
-	series, err := metrics.QueryEnergy(now.BeginningOfDay(), time.Now(), "day", true)
+	scale, err := site.solarScaleCached()
 	if err != nil {
-		site.log.ERROR.Printf("solar forecast scale: %v", err)
 		return 1
 	}
+	return scale
+}
 
-	var pv, fcst float64
-	for _, s := range series {
-		if len(s.Data) == 0 {
-			continue
+// querySolarScale does the actual metrics query and percentile calculation
+// for solarScale, given the current beginning-of-day boundary.
+func (site *Site) querySolarScale(bod time.Time) (float64, error) {
+	from := bod.AddDate(0, 0, -solarScaleWindow)
+	series, err := metrics.QueryEnergy(from, time.Now(), "day", true)
+	if err != nil {
+		return 0, err
 	}
+
+	pv := make(map[string]float64, solarScaleWindow)
+	fcst := make(map[string]float64, solarScaleWindow)
+	for _, s := range series {
+		var m map[string]float64
 		switch s.Group {
 		case metrics.PV:
-			pv = s.Data[0].Energy
+			m = pv
 		case metrics.Forecast:
-			fcst = s.Data[0].Energy
+			m = fcst
+		default:
+			continue
+		}
+		for _, d := range s.Data {
+			m[d.Start.Format("2006-01-02")] = d.Energy
 		}
 	}
 
-	const minEnergy = 0.5 // kWh
-	if fcst <= 0 || pv <= minEnergy {
-		return 1
+	today := bod.Format("2006-01-02")
+	ratios := make([]float64, 0, len(fcst))
+	for day, f := range fcst {
+		// skip today (partial) and dark days where the ratio is noise. The threshold
+		// applies to production as well: a near-zero yield against a healthy forecast
+		// is a fault (snow, soiling, inverter or metering outage), not a bias that
+		// should be projected onto the next solarScaleWindow days.
+		if p := pv[day]; day != today && f > solarScaleMinEnergy && p > solarScaleMinEnergy {
+			ratios = append(ratios, p/f)
+		}
 	}
 
-	scale := pv / fcst
-	site.log.DEBUG.Printf("solar forecast: produced %.3fkWh, forecasted %.3fkWh, scale %.3f", pv, fcst, scale)
-	return scale
+	scale, ok := percentileOf(ratios, solarScalePercentile, solarScaleMinSamples)
+	if !ok {
+		return 1, nil
+	}
+	site.log.DEBUG.Printf("solar scale P%.0f over %d days = %.3f", solarScalePercentile*100, len(ratios), scale)
+	return scale, nil
+}
+
+// percentileOf returns the p-th percentile (0..1) of values by nearest-rank on the
+// sorted series, or false when fewer than minSamples are present.
+func percentileOf(values []float64, p float64, minSamples int) (float64, bool) {
+	if len(values) < minSamples {
+		return 0, false
+	}
+	s := slices.Clone(values)
+	slices.Sort(s)
+	return s[int(p*float64(len(s)-1))], true
 }
 
 func (site *Site) isDynamicTariff(usage api.TariffUsage) bool {

core/site_tariffs_test.go

@@ -92,3 +92,49 @@ func TestTimeseriesMarshal(t *testing.T) {
 		})
 	}
 }
+
+func TestPercentileOf(t *testing.T) {
+	// n values of v
+	fill := func(n int, v float64) []float64 {
+		s := make([]float64, n)
+		for i := range s {
+			s[i] = v
+		}
+		return s
+	}
+
+	t.Run("too few samples returns false", func(t *testing.T) {
+		_, ok := percentileOf(nil, 0.5, solarScaleMinSamples)
+		assert.False(t, ok)
+
+		_, ok = percentileOf(fill(solarScaleMinSamples-1, 0.9), 0.5, solarScaleMinSamples)
+		assert.False(t, ok)
+	})
+
+	t.Run("stable cluster", func(t *testing.T) {
+		v, ok := percentileOf(fill(20, 0.9), 0.5, solarScaleMinSamples)
+		assert.True(t, ok)
+		assert.InDelta(t, 0.9, v, 0.001)
+	})
+
+	// P50 rejects outlier days for free: a broken forecast feed (recent ratio
+	// ~2.3) and a metering outage (ratio ~0.16) do not move the result as
+	// long as they stay a minority of the window.
+	t.Run("outlier days do not move P50", func(t *testing.T) {
+		ratios := fill(20, 0.9)                   // healthy installation bias
+		ratios = append(ratios, fill(4, 2.3)...)  // broken forecast feed
+		ratios = append(ratios, fill(8, 0.16)...) // metering outage
+
+		v, ok := percentileOf(ratios, 0.5, solarScaleMinSamples)
+		assert.True(t, ok)
+		assert.InDelta(t, 0.9, v, 0.001)
+	})
+
+	t.Run("higher percentile shifts toward the upper tail", func(t *testing.T) {
+		ratios := append(fill(15, 0.8), fill(15, 1.2)...)
+
+		p50, _ := percentileOf(ratios, 0.5, solarScaleMinSamples)
+		p90, _ := percentileOf(ratios, 0.9, solarScaleMinSamples)
+		assert.Less(t, p50, p90)
+	})
+}

server/http_db_metrics_handler.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/evcc-io/evcc/core/metrics"
 	"github.com/evcc-io/evcc/server/db"
+	"github.com/evcc-io/evcc/util"
 )
 
 // deleteResult reports the number of affected rows
@@ -58,6 +59,9 @@ func deleteEnergyHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// invalidate cached values (e.g. the solar scale) derived from the deleted history
+	util.ResetCached()
+
 	jsonWrite(w, deleteResult{rows})
 }