-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcache_sqlite.py
More file actions
1815 lines (1669 loc) · 86.8 KB
/
Copy pathcache_sqlite.py
File metadata and controls
1815 lines (1669 loc) · 86.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""SQLite-backed cache for Wordle scores and lookahead results."""
from __future__ import annotations
import hashlib
import logging
import sqlite3
import time
from collections import OrderedDict
from pathlib import Path
logger = logging.getLogger("wordle")
def branch_reference(branch_key: bytes) -> str:
"""Return the stable handle for one encoded branch answer set."""
return hashlib.sha1(bytes(branch_key)).hexdigest()[:12]
# Two searches reaching the same optimum sum the same terms in the same order,
# so they agree exactly; the tolerance only absorbs a value that reached the
# cache by some other route.
EXACT_SCORE_TOLERANCE = 1e-9
def exact_results_agree(stored_score, stored_max_depth,
incoming_score, incoming_max_depth) -> bool:
"""Whether two exact results for one scope are the same certificate.
Equal cost is not enough. max_depth is ancestor-visible — a parent folds
a child's worst case into its own — so two equal-cost strategies with
different worst cases are different certificates, and a parent folded from
one does not describe a subtree the other supports.
import_cache expresses this same rule in SQL, over whole tables at once;
test_the_sql_equivalence_rule_matches_the_python_one keeps the two in step.
"""
if abs(stored_score - incoming_score) > EXACT_SCORE_TOLERANCE:
return False
return stored_max_depth == incoming_max_depth
def _branch_facts_by_key(rows):
"""Group exact branch rows into (unrestricted_row, {solve_budget: row}).
Both branch tables carry the same columns, so one pass over their union
separates a branch's unrestricted result from its budget-specific ones
without the caller having to know which table a row came from.
"""
facts = {}
for row in rows:
key = bytes(row["branch_key"])
canonical, by_budget = facts.get(key, (None, None))
if by_budget is None:
by_budget = {}
if row["solve_budget"] is None:
canonical = row
else:
by_budget[row["solve_budget"]] = row
facts[key] = (canonical, by_budget)
return facts
BRANCH_RESULT_COLUMNS = (
'branch_key', 'branch_reference', 'policy', 'answer_list_id',
'solve_budget', 'best_guess', 'best_score', 'updated_at', 'max_depth')
def legacy_branch_result_select(legacy_columns):
"""Select list giving a legacy branch table the post-split column shape."""
return ', '.join(
column if column in legacy_columns else f'NULL AS {column}'
for column in BRANCH_RESULT_COLUMNS)
def present_pre_split_cache_by_scope(conn):
"""Show a pre-split cache through the split's own two tables.
A cache written before the split has no branch_best_by_policy_and_budget
— the migration that creates it runs in ScoreCache._ensure_schema, which a
read-only open deliberately skips. Every reader spanning both scopes
would then fail on a database it is meant to inspect, and inspecting an
un-migrated cache is the whole point of opening one read-only.
Both tables are supplied as TEMP views over the legacy one, split on
solve_budget. Supplying only the budget half would be worse than the
failure it replaces: the legacy canonical table holds both scopes, so
every reader would take a budget-specific result for the unrestricted
optimum and hand it out at any budget — the cross-scope reuse this
schema exists to stop.
Temp objects live outside the file, so this writes nothing — it is safe
even on a connection opened immutable — and SQLite resolves an unqualified
name against temp before main, so the readers need no special case. The
view bodies name main. explicitly; unqualified they would resolve to the
temp views themselves.
"""
if conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
('branch_best_by_policy_and_budget',)).fetchone():
return
legacy_columns = {row["name"] for row in
conn.execute("PRAGMA table_info(branch_best_by_policy)")}
selection = legacy_branch_result_select(legacy_columns)
if 'solve_budget' in legacy_columns:
unrestricted, budgeted = 'solve_budget IS NULL', 'solve_budget IS NOT NULL'
else:
# Older still: with no solve_budget column every row is an
# unrestricted result and there are no budget-specific ones.
unrestricted, budgeted = '1', '0'
for view, condition in (('branch_best_by_policy', unrestricted),
('branch_best_by_policy_and_budget', budgeted)):
conn.execute(f"""
CREATE TEMP VIEW {view} AS
SELECT {selection} FROM main.branch_best_by_policy
WHERE {condition}
""")
class CacheWriteConflict(Exception):
"""Two exact results disagree for one branch at one budget scope.
Within a scope the optimum is a single number, so a second exact write
naming a different one means the two searches cannot both be right.
Recording either would invalidate whichever ancestors folded the other,
which is the failure this schema exists to prevent — so the write is
refused instead.
"""
def answer_list_id(answer_words) -> str:
"""Return the namespace key for one answer word list."""
return hashlib.sha256("\n".join(answer_words).encode()).hexdigest()
class LRUDict:
"""Fixed-capacity LRU cache backed by an OrderedDict.
Evicts the least-recently-used entry when the capacity is reached.
All operations are O(1). When max_size is None the cache is unbounded
(identical behaviour to a plain dict, but with the move-to-end overhead
on every access — callers that want truly unbounded should pass None to
opt out of the overhead).
"""
def __init__(self, max_size=None):
self._max = max_size
self._data = OrderedDict()
def get(self, key, default=None):
if key not in self._data:
return default
self._data.move_to_end(key)
return self._data[key]
def __setitem__(self, key, value):
if key in self._data:
self._data.move_to_end(key)
self._data[key] = value
if self._max is not None and len(self._data) > self._max:
self._data.popitem(last=False)
def __getitem__(self, key):
self._data.move_to_end(key)
return self._data[key]
def __contains__(self, key):
return key in self._data
def pop(self, key, *args):
return self._data.pop(key, *args)
def pop_matching(self, predicate):
"""Drop every entry whose key satisfies predicate."""
for key in [key for key in self._data if predicate(key)]:
del self._data[key]
def __len__(self):
return len(self._data)
def _available_ram_bytes() -> int:
"""Return MemAvailable from /proc/meminfo, or 0 on any read error."""
try:
with open('/proc/meminfo') as f:
for line in f:
if line.startswith('MemAvailable:'):
return int(line.split()[1]) * 1024
except (OSError, ValueError, IndexError):
pass
return 0
def mem_cache_limit(n_workers: int, ram_fraction: float = 0.4,
bytes_per_entry: int = 250) -> int:
"""Compute a per-worker _mem_cache entry cap from available RAM.
Divides (ram_fraction * available_ram) evenly across n_workers. Falls
back to 500,000 entries if available RAM cannot be determined.
bytes_per_entry is an estimate of the Python memory cost per cache entry
(branch_key bytes blob + tuple + dict-node overhead).
"""
available = _available_ram_bytes()
if available <= 0 or n_workers <= 0:
return 500_000
return max(10_000, int(available * ram_fraction / n_workers / bytes_per_entry))
def _is_disk_io_error(exc):
"""True if exc is the transient 'disk I/O error' OperationalError that
iCloud File Provider Storage raises when a sync pass holds the lock on
the cache file or its WAL — see ScoreCache.checkpoint.
"""
return "disk I/O error" in str(exc)
class ScoreCache:
"""Persists per-word scores and branch lookahead results.
Tables:
candidate_scores — per-word scoring method results (level 1)
branch_best_by_policy — the word a search policy judged best for a
branch, and the score that earned it that
judgment (levels 2+); the "by_policy" in the
table name carries the scoping that lets the
best_guess/best_score columns stay short —
"best" is only ever read alongside the policy
that decided it
answer_list — fingerprint of the answer word set
All entries are keyed by answer_list_id so a different answer list
produces a clean namespace without needing a new file.
"""
def __init__(self, db_path, answer_words, timeout=30.0,
checkpoint_on_close=True, max_mem_entries=None,
read_only=False):
self.db_path = Path(db_path)
self.answer_words = list(answer_words)
self.read_only = read_only
# A read-only cache never checkpoints: TRUNCATE is itself a write.
self.checkpoint_on_close = checkpoint_on_close and not read_only
if read_only:
# An inspection pass over a live cache must leave no trace, not
# even the schema migration and answer-list row an ordinary open
# writes. SQLite enforces that for us: mode=ro rejects every
# write, so a caller that reaches for one gets an error instead of
# a silently swallowed no-op. It also refuses to create the file,
# which is what keeps a mistyped path from reading as an empty
# database.
self._conn = sqlite3.connect(
f"file:{self.db_path}?mode=ro", uri=True,
timeout=timeout, isolation_level=None
)
self._conn.row_factory = sqlite3.Row
present_pre_split_cache_by_scope(self._conn)
self.answer_list_id = answer_list_id(self.answer_words)
else:
self._conn = sqlite3.connect(
self.db_path, timeout=timeout, isolation_level=None
)
self._conn.row_factory = sqlite3.Row
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.execute("PRAGMA synchronous=NORMAL")
self._ensure_schema()
self.answer_list_id = self._ensure_answer_list()
self.read_hits = 0
self.read_misses = 0
self.write_count = 0
# Exact results re-derived and found already stored.
self.redundant_write_count = 0
# ...of which the stored worst case differed from the one just
# computed, so the caller adopted the stored certificate.
self.adopted_depth_count = 0
# In-memory mirror of branch_best_by_policy rows seen this session.
# Branch results are write-once/exact, so a hit here is as good as
# a SQLite hit but ~1000x cheaper — recursive ERD search re-reads the
# same small branches millions of times across sibling branches.
# max_mem_entries caps the cache size with LRU eviction so long-lived
# worker processes do not consume unbounded memory. None = unbounded.
self._mem_cache = LRUDict(max_size=max_mem_entries)
# Session mirror of proven losses (positive hits only): (branch_key,
# policy) -> largest budget at which the branch is proven a loss. A
# worker re-encounters the same inseparable residue under thousands of
# candidates within one branch sweep; this turns each repeat into an
# O(1) hit instead of a fresh exhaustive disproof.
self._loss_mem_cache = LRUDict(max_size=max_mem_entries)
def __del__(self):
conn = getattr(self, '_conn', None)
if conn is not None:
try:
conn.close()
except sqlite3.ProgrammingError:
# conn was created on a different thread than the one
# finalizing it; SQLite connections are thread-affine, so
# closing here is impossible.
pass
def _is_migration_done(self, name):
"""Return True if migration `name` has been recorded as complete."""
return self._conn.execute(
"SELECT 1 FROM schema_migrations WHERE name = ?", (name,)
).fetchone() is not None
def _mark_migration_done(self, name):
"""Record migration `name` as complete so it is skipped on future opens."""
self._conn.execute(
"INSERT OR IGNORE INTO schema_migrations (name, completed_at) VALUES (?, ?)",
(name, int(time.time()))
)
def _rename_source_summaries_to_opener(self):
# Runs before the CREATE TABLE IF NOT EXISTS statements below, so —
# unlike the older renames in this file — there is no empty
# new-named shell to drop first.
#
# The plan is re-derived from actual table/column state on every
# open rather than trusting schema_migrations alone: the table
# rename and the column rename are two statements, and a process
# that dies between them leaves a table already carrying the new
# name but still holding the old column -- a state indistinguishable
# from "not started" if this only checked the old table's presence.
# Recomputing catches that state (and a database left there by an
# earlier, non-atomic version of this migration) and finishes it
# instead of silently skipping it forever.
plan = (
('completed_source_summaries', 'completed_opener_summaries'),
('root_response_group_summaries', 'opener_response_group_summaries'),
)
tables = {row["name"] for row in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
statements = []
for old_table, new_table in plan:
old_present = old_table in tables
new_present = new_table in tables
if old_present and new_present:
raise RuntimeError(
f"both {old_table} and {new_table} exist: half-applied "
"opener-rename migration, needs manual repair")
if old_present:
statements.append(f"ALTER TABLE {old_table} RENAME TO {new_table}")
statements.append(
f"ALTER TABLE {new_table} RENAME COLUMN source_word TO opener")
elif new_present:
cols = {row["name"] for row in
self._conn.execute(f"PRAGMA table_info({new_table})")}
if "opener" in cols:
continue
if "source_word" in cols:
statements.append(
f"ALTER TABLE {new_table} RENAME COLUMN source_word TO opener")
else:
raise RuntimeError(
f"{new_table} has neither opener nor source_word: "
"unrecognized schema, needs manual repair")
# else: neither present -- CREATE TABLE IF NOT EXISTS below
# creates it directly under the current name.
if not statements:
self._mark_migration_done('rename_source_summaries_to_opener')
return
# Table rename, column rename, and the migration marker all land in
# one transaction so a crash anywhere in the sequence leaves either
# the untouched legacy schema or the fully renamed one -- never a
# table with the new name and the old column.
self._conn.execute("BEGIN IMMEDIATE")
try:
for statement in statements:
self._conn.execute(statement)
self._mark_migration_done('rename_source_summaries_to_opener')
self._conn.execute("COMMIT")
except Exception:
self._conn.execute("ROLLBACK")
raise
def _ensure_schema(self):
# Must be first: every migration guard below reads from this table.
self._conn.execute("""
CREATE TABLE IF NOT EXISTS schema_migrations (
name TEXT PRIMARY KEY,
completed_at INTEGER NOT NULL
)
""")
self._conn.execute("""
CREATE TABLE IF NOT EXISTS answer_list (
answer_list_id TEXT PRIMARY KEY,
answer_hash TEXT NOT NULL,
answer_count INTEGER NOT NULL,
created_at INTEGER NOT NULL
)
""")
self._conn.execute("""
CREATE TABLE IF NOT EXISTS response_decomposition (
guess TEXT NOT NULL,
answer_list_id TEXT NOT NULL,
patterns BLOB NOT NULL,
updated_at INTEGER NOT NULL,
PRIMARY KEY (guess, answer_list_id)
)
""")
# Old databases may have either of two predecessor table structures:
# lookahead_result(subset_key, policy, universe_id,
# best_word, best_entropy, updated_at)
# subgroup_pick(subset_key, policy, universe_id,
# picked_word, picked_score, updated_at)
# Both are intermediate schemas on the way to branch_best_by_policy.
# Upgrade them in place so their rows survive as valid cache entries.
tables = {row["name"] for row in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
if "lookahead_result" in tables and "subgroup_pick" not in tables \
and "subgroup_best_by_policy" not in tables \
and "branch_best_by_policy" not in tables:
self._conn.execute(
"ALTER TABLE lookahead_result RENAME TO subgroup_best_by_policy")
self._conn.execute(
"ALTER TABLE subgroup_best_by_policy RENAME COLUMN best_entropy TO best_score")
self._conn.execute("DROP INDEX IF EXISTS idx_lookahead")
tables = {row["name"] for row in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
if "subgroup_pick" in tables and "subgroup_best_by_policy" not in tables \
and "branch_best_by_policy" not in tables:
self._conn.execute(
"ALTER TABLE subgroup_pick RENAME TO subgroup_best_by_policy")
self._conn.execute(
"ALTER TABLE subgroup_best_by_policy RENAME COLUMN picked_word TO best_word")
self._conn.execute(
"ALTER TABLE subgroup_best_by_policy RENAME COLUMN picked_score TO best_score")
self._conn.execute("DROP INDEX IF EXISTS idx_subgroup_pick")
self._conn.execute("""
CREATE TABLE IF NOT EXISTS branch_best_by_policy (
branch_key BLOB NOT NULL,
branch_reference TEXT,
policy TEXT NOT NULL,
answer_list_id TEXT NOT NULL,
best_guess TEXT NOT NULL,
best_score REAL NOT NULL,
updated_at INTEGER NOT NULL,
PRIMARY KEY (branch_key, policy, answer_list_id)
)
""")
# A branch has two kinds of exact result, and they are different facts:
# the unrestricted optimum, and the optimum among strategies feasible
# at one remaining-depth budget. Both can be right and differ. They
# live in separate tables because one row cannot hold both: a shared
# key makes either write destroy the other, after ancestors may already
# have folded the value it displaced, and nothing records which one
# they folded.
#
# branch_best_by_policy holds only the unrestricted optima, so its
# solve_budget column is NULL on every row it now accepts. The column
# stays because dropping it would rebuild a multi-GB table to no end,
# and legacy rows are read through it.
self._conn.execute("""
CREATE TABLE IF NOT EXISTS branch_best_by_policy_and_budget (
branch_key BLOB NOT NULL,
branch_reference TEXT,
policy TEXT NOT NULL,
answer_list_id TEXT NOT NULL,
solve_budget INTEGER NOT NULL,
best_guess TEXT NOT NULL,
best_score REAL NOT NULL,
updated_at INTEGER NOT NULL,
max_depth INTEGER,
PRIMARY KEY (branch_key, policy, answer_list_id, solve_budget)
)
""")
self._conn.execute("""
CREATE INDEX IF NOT EXISTS idx_branch_best_by_policy_and_budget
ON branch_best_by_policy_and_budget(answer_list_id, policy)
""")
self._conn.execute("""
CREATE INDEX IF NOT EXISTS idx_branch_budget_updated
ON branch_best_by_policy_and_budget(answer_list_id, updated_at)
""")
self._conn.execute("""
CREATE INDEX IF NOT EXISTS idx_branch_budget_reference
ON branch_best_by_policy_and_budget(branch_reference)
""")
self._conn.execute("""
CREATE INDEX IF NOT EXISTS idx_branch_best_by_policy
ON branch_best_by_policy(answer_list_id, policy)
""")
# Covers MAX(updated_at) WHERE answer_list_id = ? — used by last_write_ts()
# on every startup. Without this index, that query scans all 3M+ rows.
self._conn.execute("""
CREATE INDEX IF NOT EXISTS idx_branch_updated
ON branch_best_by_policy(answer_list_id, updated_at)
""")
# Proven depth-limited losses: a branch with no winning strategy within
# loss_budget guesses. Distinct from branch_best_by_policy, whose
# best_guess is NOT NULL — a loss has no best guess to record. A loss
# within b guesses is also a loss within any q <= b (fewer guesses can
# only be harder), so a row is reusable for every query budget <=
# loss_budget; loss_budget holds the largest budget at which the loss is
# proven. Lets the recurring inseparable residues of a hard branch be
# proven once instead of re-swept under every candidate that produces them.
self._conn.execute("""
CREATE TABLE IF NOT EXISTS branch_loss_by_policy (
branch_key BLOB NOT NULL,
branch_reference TEXT,
policy TEXT NOT NULL,
answer_list_id TEXT NOT NULL,
loss_budget INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
PRIMARY KEY (branch_key, policy, answer_list_id)
)
""")
self._rename_source_summaries_to_opener()
self._conn.execute("""
CREATE TABLE IF NOT EXISTS completed_opener_summaries (
opener TEXT NOT NULL, policy TEXT NOT NULL,
answer_list_id TEXT NOT NULL, completed_at INTEGER NOT NULL,
elapsed_millis INTEGER, worker_millis INTEGER NOT NULL,
telemetry_epochs TEXT NOT NULL DEFAULT '',
PRIMARY KEY (opener, policy, answer_list_id)
)
""")
summary_columns = {row["name"] for row in self._conn.execute(
"PRAGMA table_info(completed_opener_summaries)")}
if "telemetry_epochs" not in summary_columns:
self._conn.execute(
"ALTER TABLE completed_opener_summaries "
"ADD COLUMN telemetry_epochs TEXT NOT NULL DEFAULT ''")
self._conn.execute("""
CREATE TABLE IF NOT EXISTS opener_response_group_summaries (
opener TEXT NOT NULL, response_pattern TEXT NOT NULL,
policy TEXT NOT NULL, answer_list_id TEXT NOT NULL,
branch_count INTEGER NOT NULL, search_node_count INTEGER NOT NULL,
worker_millis INTEGER NOT NULL, first_created_at INTEGER,
last_finalized_at INTEGER, telemetry_epochs TEXT NOT NULL,
PRIMARY KEY (opener, response_pattern, policy, answer_list_id)
)
""")
# 'subset_blob' was renamed to 'subset_key' — same encoding, cleaner
# name. Databases migrated from lookahead_result or subgroup_pick may
# still carry the old column name (in subgroup_best_by_policy before
# the rename_subgroup_to_branch migration below).
cols = {row["name"] for row in
self._conn.execute("PRAGMA table_info(subgroup_best_by_policy)")}
if cols and "subset_key" not in cols and "subset_blob" in cols: # pragma: migration
self._conn.execute(
"ALTER TABLE subgroup_best_by_policy "
"RENAME COLUMN subset_blob TO subset_key")
# max_depth: worst-case line length of best_guess's strategy. ERD is
# now depth-limited ("expected remaining depth AND a guaranteed win
# within budget"), so a cached entry is only reusable at a remaining
# budget B when max_depth <= B. Existing rows predate this and get
# NULL — read as "depth unknown", hence never budget-safe, so they're
# recomputed under the cap rather than trusted. Nullable so the
# column adds cleanly to a multi-GB file (metadata-only ALTER).
for tbl in ('subgroup_best_by_policy', 'branch_best_by_policy'):
cols = {row["name"] for row in
self._conn.execute(f"PRAGMA table_info({tbl})")}
if cols and "max_depth" not in cols:
self._conn.execute(
f"ALTER TABLE {tbl} ADD COLUMN max_depth INTEGER")
# solve_budget encodes the reuse range of a depth-limited entry:
# NULL -> untainted: the cap never excluded any candidate anywhere,
# so the value IS the unconstrained optimum. Reusable at
# any remaining budget >= max_depth.
# = b -> tainted: a sibling candidate was killed by the cap, so
# this winner is only optimal *at budget b* (one more guess
# could revive the killed sibling). Reusable only when the
# remaining budget == b.
# Legacy rows are NULL but also have NULL max_depth, so the budget-aware
# reader rejects them (unknown depth) and recomputes.
for tbl in ('subgroup_best_by_policy', 'branch_best_by_policy'):
cols = {row["name"] for row in
self._conn.execute(f"PRAGMA table_info({tbl})")}
if cols and "solve_budget" not in cols:
self._conn.execute(
f"ALTER TABLE {tbl} ADD COLUMN solve_budget INTEGER")
# ERD policy names were renamed so both axes of the (guess-universe x
# compliance-filter) selection are spelled out in the namespace
# itself — 'erd_all' named only the universe, 'erd_answers' folded
# both axes into one word, and 'erd_constrained' named neither
# explicitly. The new names are uniform: erd_<universe>_<compliance>.
# erd_all -> erd_words_unfiltered (all words, no clue filter)
# erd_answers -> erd_answers_compliant (answer list, clue-compliant)
# 'erd_constrained' has no persisted rows: hard-mode ERD is
# path-dependent and lives only in a transient MemoryScoreCache.
if not self._is_migration_done('rename_erd_policies'): # pragma: migration
for tbl in ('subgroup_best_by_policy', 'branch_best_by_policy'):
t_exists = self._conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
(tbl,)).fetchone()
if t_exists is None:
continue
for old, new in (('erd_all', 'erd_words_unfiltered'),
('erd_answers', 'erd_answers_compliant')):
exists = self._conn.execute(
f"SELECT 1 FROM {tbl} WHERE policy = ? LIMIT 1",
(old,)
).fetchone()
if exists is not None:
self._conn.execute(
f"UPDATE {tbl} SET policy = ? WHERE policy = ?",
(new, old))
self._mark_migration_done('rename_erd_policies')
# word_scores used to be keyed only by (word, method, universe_id) —
# i.e. scoped to the whole answer set, so it could only ever cache
# the very first guess of a game. Replace it with a subset-scoped
# table (mirroring branch_best_by_policy) so any remaining-word position
# that recurs gets its scores cached, not just the opening one.
old_cols = {row["name"] for row in
self._conn.execute("PRAGMA table_info(word_scores)")}
if old_cols and "subset_hash" not in old_cols: # pragma: migration
self._conn.execute("DROP TABLE word_scores")
self._conn.execute("""
CREATE TABLE IF NOT EXISTS candidate_scores (
subset_hash TEXT NOT NULL,
word TEXT NOT NULL,
method TEXT NOT NULL,
score REAL NOT NULL,
answer_list_id TEXT NOT NULL,
updated_at INTEGER NOT NULL,
PRIMARY KEY (subset_hash, method, answer_list_id, word)
)
""")
# Legacy rows may carry the method key 'minimax' — an earlier name for
# the MAX_GROUP_SIZE scoring method that named the search strategy
# rather than the metric, making rows uninterpretable without external
# context. Rewrite them to 'max_group_size' so the database is
# self-describing. Checked
# via existence-first LIMIT 1 (see _purge_legacy_rows) so a table with
# no such rows — the steady state once this has run once — costs only
# a single indexed-or-not probe, not a full scan, on every connection.
if not self._is_migration_done('rename_method_minimax'):
for tbl in ('word_scores', 'candidate_scores'):
t_exists = self._conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
(tbl,)).fetchone()
if t_exists is None:
continue
stale_method = self._conn.execute(
f"SELECT 1 FROM {tbl} WHERE method = 'minimax' LIMIT 1"
).fetchone()
if stale_method is not None:
self._conn.execute(
f"UPDATE {tbl} SET method = 'max_group_size'"
" WHERE method = 'minimax'")
self._mark_migration_done('rename_method_minimax')
# Rename subgroup_best_by_policy -> branch_best_by_policy, and columns:
# subset_key -> branch_key
# best_word -> best_guess
# universe_id -> answer_list_id
if not self._is_migration_done('rename_subgroup_to_branch'): # pragma: migration
tables = {row["name"] for row in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
if "subgroup_best_by_policy" in tables:
# The CREATE TABLE IF NOT EXISTS above may have created an empty
# branch_best_by_policy shell; drop it before the rename so we
# don't get a "table already exists" conflict.
self._conn.execute("DROP TABLE IF EXISTS branch_best_by_policy")
self._conn.execute("DROP INDEX IF EXISTS idx_branch_best_by_policy")
self._conn.execute("DROP INDEX IF EXISTS idx_branch_updated")
self._conn.execute(
"ALTER TABLE subgroup_best_by_policy RENAME TO branch_best_by_policy")
cols = {row["name"] for row in
self._conn.execute("PRAGMA table_info(branch_best_by_policy)")}
if cols and "subset_key" in cols:
self._conn.execute(
"ALTER TABLE branch_best_by_policy "
"RENAME COLUMN subset_key TO branch_key")
if cols and "best_word" in cols:
self._conn.execute(
"ALTER TABLE branch_best_by_policy "
"RENAME COLUMN best_word TO best_guess")
self._mark_migration_done('rename_subgroup_to_branch')
# Rename word_scores -> candidate_scores
if not self._is_migration_done('rename_word_scores'): # pragma: migration
tables = {row["name"] for row in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
if "word_scores" in tables:
# Drop the empty candidate_scores shell created by CREATE TABLE
# IF NOT EXISTS before renaming the old table into its place.
self._conn.execute("DROP TABLE IF EXISTS candidate_scores")
self._conn.execute("ALTER TABLE word_scores RENAME TO candidate_scores")
self._mark_migration_done('rename_word_scores')
# Rename universe -> answer_list and universe_id -> answer_list_id
# in all tables that carry it.
if not self._is_migration_done('rename_universe_to_answer_list'): # pragma: migration
tables = {row["name"] for row in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
if "universe" in tables:
# Drop the empty answer_list shell from CREATE TABLE IF NOT EXISTS.
self._conn.execute("DROP TABLE IF EXISTS answer_list")
self._conn.execute("ALTER TABLE universe RENAME TO answer_list")
for tbl, old_col in [
('answer_list', 'universe_id'),
('response_decomposition', 'universe_id'),
('branch_best_by_policy', 'universe_id'),
('candidate_scores', 'universe_id'),
]:
cols = {row["name"] for row in
self._conn.execute(f"PRAGMA table_info({tbl})")}
if cols and old_col in cols:
self._conn.execute(
f"ALTER TABLE {tbl} RENAME COLUMN {old_col} TO answer_list_id")
self._mark_migration_done('rename_universe_to_answer_list')
# All valid 5-letter words are ASCII, so a null byte identifies the
# old null-separated branch-key encoding.
self._purge_legacy_rows("instr(branch_key, char(0)) > 0", (),
migration_name='purge_null_sep_keys')
# 'erd' was renamed to 'erd_answers' and then superseded by 'erd_all'.
self._purge_legacy_rows("policy = ?", ('erd',),
migration_name='purge_policy_erd')
# 'erd_hard' was renamed to 'erd_constrained'; constraint-compliant
# mode is now always transient (MemoryScoreCache), so any persisted
# rows under either name are useless regardless of age.
self._purge_legacy_rows("policy = ?", ('erd_hard',),
migration_name='purge_policy_erd_hard')
if not self._is_migration_done('add_branch_references'):
for table_name in ('branch_best_by_policy', 'branch_loss_by_policy'):
columns = {row["name"] for row in self._conn.execute(
f"PRAGMA table_info({table_name})")}
if "branch_reference" not in columns:
self._conn.execute(
f"ALTER TABLE {table_name} ADD COLUMN branch_reference TEXT")
rows = self._conn.execute(
f"SELECT rowid, branch_key FROM {table_name} "
"WHERE branch_reference IS NULL"
).fetchall()
self._conn.executemany(
f"UPDATE {table_name} SET branch_reference = ? WHERE rowid = ?",
[(branch_reference(row["branch_key"]), row["rowid"])
for row in rows],
)
self._conn.execute(
f"CREATE INDEX IF NOT EXISTS idx_{table_name}_reference "
f"ON {table_name}(branch_reference)"
)
self._mark_migration_done('add_branch_references')
# Budget-specific results used to share the canonical table's key, so
# a branch's two facts overwrote one another. Move them to the table
# that can hold both.
if not self._is_migration_done('split_budget_specific_branch_results'): # pragma: migration
self._conn.execute("""
INSERT OR IGNORE INTO branch_best_by_policy_and_budget
(branch_key, branch_reference, policy, answer_list_id,
solve_budget, best_guess, best_score, updated_at, max_depth)
SELECT branch_key, branch_reference, policy, answer_list_id,
solve_budget, best_guess, best_score, updated_at, max_depth
FROM branch_best_by_policy
WHERE solve_budget IS NOT NULL
""")
self._conn.execute(
"DELETE FROM branch_best_by_policy WHERE solve_budget IS NOT NULL")
self._mark_migration_done('split_budget_specific_branch_results')
# candidate_erd_by_policy memoised a candidate's folded ERD at a
# branch, keyed by a hash of the branch's word set. Given a branch
# result there was no way to ask which folds had read it, so deleting
# one — a repair, a reverification, a requeue — left every fold over it
# asserting a candidate complete whose groups were gone. The fold is
# now derived on each read from branch results already in memory, so
# the table is derived data with no reader: drop it outright rather
# than carry rows nothing consults.
if not self._is_migration_done('drop_candidate_erd_memo'): # pragma: migration
self._conn.execute("DROP TABLE IF EXISTS candidate_erd_by_policy")
self._mark_migration_done('drop_candidate_erd_memo')
def _purge_legacy_rows(self, where, params, migration_name=None):
"""One-time cleanup of stale branch_best_by_policy rows.
Once a legacy batch is gone it stays gone, so a full-table DELETE on
every connection open (including each ERDSolver thread) would scan
the whole table for nothing. Check existence first — LIMIT 1 lets
SQLite stop at the first match — and only DELETE when there's
actually something to remove.
migration_name: if given, skip the entire check on future opens once
it has been recorded as done in schema_migrations.
"""
if migration_name and self._is_migration_done(migration_name):
return
exists = self._conn.execute(
f"SELECT 1 FROM branch_best_by_policy WHERE {where} LIMIT 1", params
).fetchone()
if exists is not None:
self._conn.execute(
f"DELETE FROM branch_best_by_policy WHERE {where}", params)
if migration_name: # pragma: migration
self._mark_migration_done(migration_name)
def _ensure_answer_list(self):
list_id = answer_list_id(self.answer_words)
now = int(time.time())
self._conn.execute("""
INSERT OR IGNORE INTO answer_list
(answer_list_id, answer_hash, answer_count, created_at)
VALUES (?, ?, ?, ?)
""", (list_id, list_id, len(self.answer_words), now))
return list_id
def close(self):
if self.checkpoint_on_close:
self.checkpoint()
self._conn.close()
def checkpoint(self):
"""Fold the WAL into the main database file (PRAGMA wal_checkpoint(TRUNCATE)).
Leaves wordle_cache.sqlite3 self-contained with no -wal/-shm
sidecars, so it's always safe to copy off-device - and the latest
writes survive even if iOS suspends/kills the process without a
clean close().
This is an optimization, not a durability requirement: every write
is already committed to the WAL, so a failed checkpoint loses
nothing. On iOS the cache file lives under iCloud's File Provider
Storage, where a sync pass can transiently hold the exclusive lock
TRUNCATE needs - swallow that rather than letting it take down a
background solver thread (or close()) over a no-op.
"""
try:
self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except sqlite3.OperationalError as exc:
logger.warning("wal_checkpoint(TRUNCATE) failed: %s", exc)
# ------------------------------------------------------------------
# Branch lookahead cache (levels 2+)
# ------------------------------------------------------------------
@staticmethod
def encode_subset(words):
"""Canonical key for a set of words: sorted, concatenated, no separator.
All Wordle words are exactly 5 ASCII characters, so a key of length 5N
encodes exactly N words recoverable by slicing at fixed 5-byte offsets.
"""
return "".join(sorted(words)).encode("utf-8")
def read(self, branch_key, policy):
"""Return (best_guess, best_score) or None on cache miss.
best_guess is whichever word this policy's search judged best for
this branch — judged by that policy's own metric, not some
universal notion of "best". best_score is that metric's value for
best_guess, and its meaning is policy-dependent: an entropy in bits
(higher is better) for lookahead policies ('full'/'hard'), or an
expected-remaining-guesses cost (lower is better) for ERD policies
('erd_words_unfiltered'/'erd_answers_compliant'). Callers that care
about the number must already know which policy they asked for — the
table name (branch_best_by_policy) and its policy column carry that
scoping, so the columns themselves can stay "best_guess"/"best_score"
without re-litigating it.
"""
cached = self._mem_cache.get((branch_key, policy, None))
if cached is not None:
self.read_hits += 1
return cached[:2]
full = self._read_full(branch_key, policy)
if full is None:
return None
return full[:2]
def read_with_depth(self, branch_key, policy):
"""Like read(), but returns (best_guess, best_score, max_depth, solve_budget).
Answers for the branch's *unrestricted* result only — the optimum over
all strategies, reusable at any budget its own max_depth can meet.
solve_budget is therefore None on every row this returns except a
legacy one, and a legacy row (max_depth None) is never budget-reusable.
A search under a cap wants read_for_budget, which consults the
budget-specific results too.
"""
cached = self._mem_cache.get((branch_key, policy, None))
if cached is not None:
self.read_hits += 1
return cached
return self._read_full(branch_key, policy)
def _read_stored_row(self, branch_key, policy, solve_budget):
"""One scope's row straight from SQLite, with no session mirror.
The mirror can hold a value this connection read before another wrote,
so anything reconciling against what is *durably* stored — the write
path's conflict check — has to come here rather than through the
cached reads.
"""
if solve_budget is None:
row = self._conn.execute("""
SELECT best_guess, best_score, max_depth, solve_budget
FROM branch_best_by_policy
WHERE branch_key = ? AND policy = ? AND answer_list_id = ?
""", (branch_key, policy, self.answer_list_id)).fetchone()
else:
row = self._conn.execute("""
SELECT best_guess, best_score, max_depth, solve_budget
FROM branch_best_by_policy_and_budget
WHERE branch_key = ? AND policy = ? AND answer_list_id = ?
AND solve_budget = ?
""", (branch_key, policy, self.answer_list_id,
solve_budget)).fetchone()
if row is None:
return None
return (row["best_guess"], row["best_score"],
row["max_depth"], row["solve_budget"])
def _read_full(self, branch_key, policy):
result = self._read_stored_row(branch_key, policy, None)
if result is None:
self.read_misses += 1
return None
self.read_hits += 1
self._mem_cache[(branch_key, policy, None)] = result
return result
def _read_full_at_budget(self, branch_key, policy, budget):
"""The exact result stored for `branch_key` at remaining budget `budget`.
A budget-specific row is valid only at the budget it was solved under,
so it is looked up by that budget rather than filtered after the fact.
"""
cached = self._mem_cache.get((branch_key, policy, budget))
if cached is not None:
self.read_hits += 1
return cached
result = self._read_stored_row(branch_key, policy, budget)
if result is None:
self.read_misses += 1
return None
self.read_hits += 1
self._mem_cache[(branch_key, policy, budget)] = result
return result
def read_for_budget(self, branch_key, policy, budget):
"""The entry a search at `budget` should reuse, or None.
The unrestricted result wins whenever its strategy fits: it is
globally optimal, so it is also optimal within any budget its own
worst case can meet. Only when it does not fit is the budget-specific
result consulted, and only the one solved at exactly this budget — a
row from another budget is optimal against a different set of feasible
strategies and is not an exact hit here. An unlimited search
(budget None) reads the unrestricted table alone.
Returns the same (best_guess, best_score, max_depth, solve_budget)
tuple the plain reads return, so `wordle_engine._cache_reuse` remains
the one place the reuse rule is stated.
"""
canonical = self.read_with_depth(branch_key, policy)
if budget is None:
return canonical
if canonical is not None:
max_remaining_depth = canonical[2]
if max_remaining_depth is not None and max_remaining_depth <= budget:
return canonical
return self._read_full_at_budget(branch_key, policy, budget)
def reset_read_counters(self):
self.read_hits = 0
self.read_misses = 0
def write(self, branch_key, policy, best_guess, best_score,
max_depth=None, solve_budget=None):
"""Store the word a policy's search judged best for a branch, its
score, and (for depth-limited ERD) the worst-case line length of that
strategy plus its reuse-range marker. max_depth=None marks a
legacy/unbudgeted write. solve_budget routes the result: None is the
unrestricted optimum and goes to branch_best_by_policy; an int is the
optimum under that cap and goes to branch_best_by_policy_and_budget.
The two are separate facts and neither displaces the other.
A result already stored for the same branch at the same scope is kept
rather than replaced, and **returned**: the caller must adopt it before
folding anything, because what a solver hands its parent has to be what
the cache durably holds. Equal-cost strategies can differ in
max_depth, which is ancestor-visible, so a caller that kept its own
worst case would fold a parent the stored child does not support —
the inconsistent ancestry this schema exists to prevent, reached
without any overwrite.
Returns the durable (best_guess, best_score, max_depth, solve_budget).
A second result that disagrees on the *cost* cannot be reconciled by
adoption — both claim to be the optimum, so one of them is wrong — and
raises CacheWriteConflict.
A transient 'disk I/O error' (e.g. iCloud File Provider Storage
holding the cache file's lock during a sync pass — see checkpoint())
is logged and swallowed rather than propagated: this runs at every
level of a min_expected_guesses recursion, so letting it raise would
unwind the entire call stack and abort the background solver thread,
discarding every result computed this run — not just this one.
best_guess/best_score are still recorded in _mem_cache so this run's
recursion keeps the memoization benefit even when the on-disk write
fails; the row is simply recomputed on a later run.
"""
now = int(time.time())
entry = (best_guess, best_score, max_depth, solve_budget)
if solve_budget is None:
table = 'branch_best_by_policy'
conflict_target = 'branch_key, policy, answer_list_id'
else:
table = 'branch_best_by_policy_and_budget'
conflict_target = 'branch_key, policy, answer_list_id, solve_budget'
try:
# Creating the row IS the check. A read followed by an insert
# leaves a window two workers both pass through, and the second
# insert would then displace a result an ancestor may already have