Version 2026.03.0

dao/.dockerignore

@@ -1 +1,2 @@
 tests/
+pred/

dao/CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog 刀 DAO
 # Day Ahead Optimizer
+# 2026.03.0
+### New features:
+- Add Fast Forward and Fast Reverse to web interface Home page (PR from @tomvandepoel3)
+- Improve error handling. Got "could not convert string to float: unavailable" exception without a reference to the problem HA entity. 
+This change should help locate and fix such issues.
+- All used data of the ml-training are output in debug-level of logging (take care much data!!)
+### Change
+Changed entity hp heat demand from input_boolean (values "on"/"off" to input_select values "off"/"eco"/"max"(="on")
+
+### Fixes:
+- Corrected logging when there are no data in wp-sensor(s)
+- Fixed error when retrieving  wp-data (reported by @rescla)
+- Fix error only supply zero's for missing sensor data of the solar inverter after the first record.
+- Fixed error when checking runtime hours heatpump and there no data (reported by @rescla)
+- Fixed error and better warning when no data for actual soc level battery (reported by @tonvanboven)
+- Fixed error when optional "entity calculated end" (machine) is not defined (reported by @Xelaph)
+- Fixed error when hp-stages are not sorted ascending max_power (reported by @Mvdw)
+- Update several python modules
+- Fixed error report/api with period "morgen"
+- Fixed error "reduce hours" with interval "1hour" (pr by @bramgradussen)
+- Fixed error missing inverter values at the begin/end of period (@reported by @DaBit)
+- Fixed error when reducing power during charging at high soc and during discharging 
+at low soc, taken the mean value of the soc at the start and the soc at the end of the interval (reported by @bartzzz)
+- Fixed error with flex setting of "dc_to_bat max power" or "bat_to_dc max power" (reported by @DaBit)
+- Fixed error not planning heatpump in first interval when not in run-mode (reported by @f.welvering)
+- Missing hour-values (solar-inverters) are filled up by zero's (suggested by @DaBit)
+- Fixed error when "-" is used in name of solar-devices (reported by @patrickvorgers and @Asclepius8)
+- Made optional battery settings "bat_to_dc max power" and "dc_to_bat max power" flex-setting (feature request by @DaBit)
+- Reduce power during charging at high soc and during discharging at low soc (feature requests form @bartzzz and @arjenhiemstra)
+- Made check 'optimal lower level" lower as "lower limit" (feature request of @mistral2)
+
 # 2026.02.2
 - Fix error in calculating heating window boiler
 - Fixed error in reports and api with interval "vandaag en morgen"
@@ -17,7 +48,7 @@
 Added missing module tzdata
 
 ### Breaking change
-The file-format ofthe calculated model is changed (update of module pandas).
+The file-format of the calculated model is changed (update of module pandas).
 The ml_prediction works only after a new training of the models.<br>
 ### Changes:
 - Update several python modules
@@ -211,7 +242,7 @@ Fix error api prognose pv_dc
 - You can configure the meteo-model for your data (option, default **harmonie**) 
 - You can configure the max number of attempts (option, default 2)<br>
 More info in DOCS.md
-- Fixed index-error when more than one batteries are used (reported by @PSMGoossens)
+- Fixed index-error when more than one batterie are used (reported by @PSMGoossens)
 - Improved graphical presentation received meteodata
 - Improved logging getting meteodata
 - Fixed error handling getting meteo-data

dao/config.yaml

@@ -1,6 +1,6 @@
 ---
 name: 刀 Day Ahead Optimizer
-version: 2026.02.2
+version: 2026.03.0
 slug: day_ahead_opt
 description: Home Assistant Community Add-ons for day ahead optimizations
 url: https://github.com/corneel27/day-ahead

dao/prog/da_config.py → dao/lib/da_config.py

@@ -2,8 +2,10 @@
 import logging
 import os
 
+from pandas.io.common import file_exists
+
 # from logging import raiseExceptions
-from dao.prog.db_manager import DBmanagerObj
+from dao.lib.db_manager import DBmanagerObj
 import sqlalchemy_utils
 
 
@@ -24,7 +26,10 @@ def __init__(self, file_name: str):
         self.options = self.parse(file_name)
         datapath = os.path.dirname(file_name)
         file_secrets = datapath + "/secrets.json"
-        self.secrets = self.parse(file_secrets)
+        if file_exists(file_secrets):
+            self.secrets = self.parse(file_secrets)
+        else:
+            self.secrets = {}
 
     def get(
         self, keys: list, options: dict = None, default=None

dao/prog/da_meteo.py → dao/lib/da_meteo.py

@@ -8,9 +8,9 @@
 from requests import get
 import matplotlib.pyplot as plt
 import knmi
-from dao.prog.da_graph import GraphBuilder
-from dao.prog.da_config import Config
-from dao.prog.db_manager import DBmanagerObj
+from dao.lib.da_graph import GraphBuilder
+from dao.lib.da_config import Config
+from dao.lib.db_manager import DBmanagerObj
 from sqlalchemy import Table, select, func, and_
 
 

dao/prog/da_prices.py → dao/lib/da_prices.py

@@ -1,18 +1,16 @@
-from dao.prog.da_config import Config
+from dao.lib.da_config import Config
 import pandas as pd
-from dao.prog.db_manager import DBmanagerObj
+from dao.lib.db_manager import DBmanagerObj
 from entsoe import EntsoePandasClient
 import datetime
 import sys
 from requests import get, post
 from nordpool.elspot import Prices
 import pytz
-import tzdata
 import json
 import math
 import pprint as pp
 import logging
-from sqlalchemy import Table, select, and_
 
 
 class DaPrices:

dao/pred/check_db_dap.py

@@ -0,0 +1,216 @@
+import datetime
+
+from sqlalchemy import (
+    Table,
+    Column,
+    Integer,
+    DateTime,
+    String,
+    BigInteger,
+    Float,
+    ForeignKey,
+    UniqueConstraint,
+    select,
+    desc,
+    insert,
+    update,
+    and_,
+    delete,
+    literal_column,
+)
+import pandas as pd
+
+#  from da_base import DaBase
+#  sys.path.append("../")
+from dao.lib.da_config import Config
+from version import __version__
+from utils import version_number
+
+
+class CheckDAPDB:
+    def __init__(self, file_name: str | None = None):
+        self.file_name = file_name
+        self.config = Config(self.file_name)
+        self.version = __version__
+        self.last_version = None
+        self.db_da = self.config.get_db_da(key="database_dap", check_create=True)
+        self.engine = self.db_da.engine
+
+    def upsert_variabel(self, variabel_table, record):
+        select_variabel = select(variabel_table.c.id).where(
+            variabel_table.c.id == record[0]
+        )
+        with self.engine.connect() as connection:
+            variabel_result = connection.execute(select_variabel).first()
+        if variabel_result:
+            query = (
+                update(variabel_table)
+                .where(variabel_table.c.id == record[0])
+                .values(code=record[1], name=record[2], dim=record[3])
+            )
+        else:
+            query = insert(variabel_table).values(
+                id=record[0], code=record[1], name=record[2], dim=record[3]
+            )
+        with self.engine.connect() as connection:
+            connection.execute(query)
+            connection.commit()
+        return
+
+    def get_all_var_data(
+        self,
+        tablename: str,
+        column_name: str,
+    ):
+        """
+        Retourneert een dataframe
+        :param tablename: de naam van de tabel "prognoses" of "values"
+        :param column_name: de code van het veld
+        :return:
+        """
+
+        variabel_table = Table(
+            "variabel", self.db_da.metadata, autoload_with=self.engine
+        )
+        values_table = Table(tablename, self.db_da.metadata, autoload_with=self.engine)
+        query = select(
+            values_table.c.time.label("time"),
+            literal_column("'" + column_name + "'").label("code"),
+            values_table.c.value.label("value"),
+        ).where(
+            and_(
+                variabel_table.c.code == column_name,
+                values_table.c.variabel == variabel_table.c.id,
+            )
+        )
+        query = query.order_by("time")
+
+        with self.engine.connect() as connection:
+            df = pd.read_sql(query, connection)
+        return df
+
+    def delete_all_var_data(
+        self,
+        tablename: str,
+        variabel_id: int,
+    ):
+        values_table = Table(tablename, self.db_da.metadata, autoload_with=self.engine)
+        delete_stmt = delete(values_table).where(
+            values_table.c.variabel == variabel_id,
+        )
+        with self.engine.connect() as connection:
+            connection.execute(delete_stmt)
+            connection.commit()
+        return
+
+    def update_db_da(self):
+        # Defining the Engine
+        # Create the Metadata Object
+        metadata = self.db_da.metadata
+        # Define the version table
+        version_table = Table(
+            "version",
+            metadata,
+            Column("id", Integer, primary_key=True, autoincrement=True),
+            Column("moment", DateTime, unique=True),
+            Column("value", String(20), unique=True),
+        )
+        # Create the version table (if not exists)
+        metadata.create_all(self.engine)
+        l_version = 20251201
+
+        query = select(version_table.c.moment, version_table.c.value).order_by(
+            desc(version_table.c.moment)
+        )
+        with self.engine.connect() as connection:
+            rows = pd.read_sql(query, connection)
+        if len(rows) >= 1:
+            self.last_version = rows.iloc[0]["value"]
+            l_version = version_number(self.last_version)
+        n_version = version_number(self.version)
+
+        variabel_tabel = Table(
+            "variabel",
+            metadata,
+            Column("id", Integer, primary_key=True, autoincrement=True),
+            Column("code", String(15), unique=True, nullable=False),
+            Column("name", String(50), unique=True, nullable=False),
+            Column("dim", String(10), nullable=False),
+            sqlite_autoincrement=True,  # Ensure SQLite uses AUTOINCREMENT
+        )
+
+        if l_version <= 20260101:
+            # check variabel
+            # Create the version table (if not exists)
+            variabel_tabel.create(self.engine)
+            records = [
+                [1, "cons", "Verbruik", "MWh"],
+                [2, "prod_zon", "Productie zon", "MWh"],
+                [3, "prod_wind", "Productie wind (land)", "MWh"],
+                [4, "prod_zeewind", "Productie wind (zee)", "MWh"],
+                [5, "da", "Day Ahead prijs epex", "euro/kWh"],
+            ]
+
+            for i in range(len(records)):
+                record = records[i]
+                self.upsert_variabel(variabel_tabel, record)
+
+            print('Table "variabel" met inhoud gecreeerd.')
+
+            # table "values" maken
+            values_tabel = Table(
+                "values",
+                metadata,
+                Column("id", Integer, primary_key=True, autoincrement=True),
+                Column(
+                    "variabel",
+                    Integer,
+                    ForeignKey("variabel.id", ondelete="CASCADE"),
+                    nullable=False,
+                ),
+                Column("time", BigInteger, nullable=False),
+                Column("value", Float),
+                UniqueConstraint("variabel", "time"),
+                sqlite_autoincrement=True,  # Ensure SQLite uses AUTOINCREMENT
+            )
+            values_tabel.create(self.engine)
+
+            print('Table "values" gecreeerd.')
+            prognoses_tabel = Table(
+                "prognoses",
+                metadata,
+                Column("id", Integer, primary_key=True, autoincrement=True),
+                Column(
+                    "variabel",
+                    Integer,
+                    ForeignKey("variabel.id", ondelete="CASCADE"),
+                    nullable=False,
+                ),
+                Column("time", BigInteger, nullable=False),
+                Column("value", Float),
+                UniqueConstraint("variabel", "time"),
+                sqlite_autoincrement=True,  # Ensure SQLite uses AUTOINCREMENT
+            )
+            prognoses_tabel.create(self.engine)
+            print('Table "prognoses" gecreeerd.')
+
+        if l_version < n_version:
+            # update version number database
+            moment = datetime.datetime.fromtimestamp(
+                round(datetime.datetime.now().timestamp())
+            )
+            insert_query = insert(version_table).values(
+                moment=moment, value=self.version
+            )
+            with self.engine.connect() as connection:
+                connection.execute(insert_query)
+                connection.commit()
+
+
+def main():
+    checkdb = CheckDAPDB("options_dap.json")
+    checkdb.update_db_da()
+
+
+if __name__ == "__main__":
+    main()