Skip to content

Commit 9b0f1c5

Browse files
WorkingRobotWorkingRobot
committed
No more pending fields
1 parent cd1c78d commit 9b0f1c5

9 files changed

Lines changed: 65 additions & 104 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,6 @@ jobs:
5050
runs-on: ubuntu-latest
5151
needs: build
5252
if: startsWith(github.ref, 'refs/tags/')
53-
env:
54-
DOTNET_CLI_TELEMETRY_OPTOUT: true
5553

5654
steps:
5755
- name: Download Linux Build

‎.github/workflows/test.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ jobs:
3131
run: dotnet build -c Release
3232

3333
- name: Retrieve cache
34-
id: exd-cache
3534
uses: actions/cache@v4
3635
with:
3736
path: data
@@ -40,7 +39,7 @@ jobs:
4039
exd-
4140
4241
- name: Download Game Data
43-
uses: WorkingRobot/ffxiv-downloader@v5
42+
uses: WorkingRobot/ffxiv-downloader@v7
4443
with:
4544
output-path: data
4645
regex: '^sqpack\/ffxiv\/0a0000\..+$'

‎BreakingValidators/FieldNamesAndTypes.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
namespace EXDTooler.BreakingValidators;
66

7+
[Obsolete("Pending fields are no longer used, but this may be useful in the future")]
78
public sealed class FieldNamesAndTypes : IBreakingValidator<FieldNamesAndTypes>
89
{
910
private FieldNamesAndTypes() { }
@@ -47,7 +48,7 @@ private static void ValidateFields(List<Field> baseFields, List<Field> newFields
4748
throw new ValidationException($"Field '{field.Name}' cannot remove targets ({string.Join(", ", targetList)})");
4849
}
4950
}
50-
51+
5152
if (baseFields.Count != newFields.Count)
5253
throw new ValidationException("Field count mismatch");
5354
}

‎BreakingValidators/IBreakingValidator.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
namespace EXDTooler.BreakingValidators;
55

6+
[Obsolete("Pending fields are no longer used, but this may be useful in the future")]
67
public interface IBreakingValidator<T> where T : IBreakingValidator<T>
78
{
89
abstract static void Validate(Sheet baseSheet, Sheet newSheet, IReadOnlyList<ExcelColumnDefinition> cols, ColDefReader colDefs);

‎ColDefReader.cs‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@
1010

1111
namespace EXDTooler;
1212

13-
public sealed class ColDefReader(ImmutableSortedDictionary<string, List<ExcelColumnDefinition>> dict)
13+
public sealed class ColDefReader(ImmutableSortedDictionary<string, ExcelColumnDefinition[]> dict,
14+
ImmutableSortedSet<string> subrows)
1415
{
15-
public ImmutableSortedDictionary<string, List<ExcelColumnDefinition>> Sheets { get; } = dict;
16+
public ImmutableSortedDictionary<string, ExcelColumnDefinition[]> Sheets { get; } = dict;
17+
private ImmutableSortedSet<string> SubrowSheets { get; } = subrows;
1618

1719
private byte[]? hash;
1820
public byte[] Hash => hash ??= CalcHash();
@@ -32,29 +34,49 @@ public static ColDefReader FromColumnFile(string file)
3234
var deserializer = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance).Build();
3335

3436
using var f = File.OpenText(file);
35-
var sheets = deserializer.Deserialize<Dictionary<string, List<ExcelColumnDefinition>>>(f);
36-
return new(sheets.ToImmutableSortedDictionary());
37+
var sheets = deserializer.Deserialize<Dictionary<string, ExcelColumnDefinition[]>>(f);
38+
39+
var subrowSheets = sheets.Where(sheet => sheet.Key.EndsWith("@Subrow"))
40+
.Select(sheet => sheet.Key[..^7])
41+
.ToImmutableSortedSet();
42+
foreach (var sheet in subrowSheets)
43+
{
44+
sheets[sheet] = sheets[$"{sheet}@Subrow"];
45+
Console.WriteLine(sheet);
46+
sheets.Remove($"{sheet}@Subrow");
47+
}
48+
49+
if (subrowSheets.Count == 0)
50+
{
51+
throw new InvalidOperationException("No subrow sheets found in the provided columns file.");
52+
}
53+
54+
return new(sheets.ToImmutableSortedDictionary(), subrowSheets);
3755
}
3856

3957
public static ColDefReader FromGameData(string gamePath)
4058
{
4159
Log.Verbose("Loading game data");
4260
using var gameData = new GameData(gamePath, new LuminaOptions()
4361
{
44-
CacheFileResources = false
62+
CacheFileResources = false,
63+
LoadMultithreaded = true
4564
});
4665

66+
var files = gameData.Excel.SheetNames
67+
.Where(p => !p.Contains('/'))
68+
.Select(sheetName => (sheetName, gameData.GetFile<ExcelHeaderFile>($"exd/{sheetName}.exh")!));
69+
4770
return new(
48-
gameData.Excel.SheetNames
49-
.Where(p => !p.Contains('/'))
50-
.Select(sheetName => (sheetName, gameData.GetFile<ExcelHeaderFile>($"exd/{sheetName}.exh")!))
51-
.Select(pair => KeyValuePair.Create(pair.sheetName, pair.Item2.ColumnDefinitions.ToList()))
52-
.ToImmutableSortedDictionary()
71+
files.ToImmutableSortedDictionary(pair => pair.sheetName, pair => pair.Item2.ColumnDefinitions),
72+
[.. files.Where(pair => pair.Item2.Header.Variant == ExcelVariant.Subrows).Select(pair => pair.sheetName)]
5373
);
5474
}
5575

76+
public ExcelColumnDefinition[] this[string sheetName] => Sheets[sheetName];
77+
5678
public uint GetColumnsHash(string sheetName) =>
57-
Crc32.Get(MemoryMarshal.AsBytes(CollectionsMarshal.AsSpan(Sheets[sheetName])));
79+
Crc32.Get(MemoryMarshal.AsBytes(Sheets[sheetName].AsSpan()));
5880

5981
public void WriteTo(TextWriter writer)
6082
{
@@ -66,7 +88,7 @@ public void WriteTo(TextWriter writer)
6688
.EnsureRoundtrip()
6789
.Build();
6890

69-
schemaSerializer.Serialize(writer, Sheets);
91+
schemaSerializer.Serialize(writer, Sheets.ToImmutableSortedDictionary(pair => $"{pair.Key}{(SubrowSheets.Contains(pair.Key) ? "@Subrow" : string.Empty)}", pair => pair.Value));
7092
}
7193

7294
private byte[] CalcHash()
@@ -85,6 +107,11 @@ private byte[] CalcHash()
85107
}
86108
}
87109

110+
foreach (var subrow in SubrowSheets)
111+
{
112+
w.Write(subrow);
113+
}
114+
88115
w.Flush();
89116
}
90117

‎Schema.cs‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ public record Sheet
1111

1212
public required List<Field> Fields { get; set; }
1313

14-
public List<Field>? PendingFields { get; set; }
15-
1614
public RelationsCollection? Relations { get; set; }
1715
}
1816

@@ -21,8 +19,6 @@ public record Field
2119
[YamlMember(Order = 0)]
2220
public string? Name { get; set; }
2321

24-
public string? PendingName { get; set; }
25-
2622
[DefaultValue(FieldType.Scalar)]
2723
public FieldType Type { get; set; }
2824

‎ValidateCommand.cs‎

Lines changed: 20 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -235,43 +235,28 @@ private static bool Validate(string sheetFile, ColDefReader colDefs, IDeserializ
235235
Validate<SheetRefs>(sheet, cols, colDefs),
236236
];
237237

238-
var pendingChecks = checks;
239-
if (sheet.PendingFields != null)
240-
{
241-
var pendingSheet = sheet with { Fields = sheet.PendingFields };
242-
pendingChecks = [
243-
Validate<ColumnCount>(pendingSheet, cols, colDefs, true),
244-
Validate<ColumnTypes>(pendingSheet, cols, colDefs, true),
245-
true,
246-
Validate<LinkConditionType>(pendingSheet, cols, colDefs, true),
247-
Validate<LinkSwitchField>(pendingSheet, cols, colDefs, true),
248-
Validate<Relations>(pendingSheet, cols, colDefs, true),
249-
Validate<SheetRefs>(pendingSheet, cols, colDefs, true),
250-
];
251-
}
252-
253-
if (checks.Any(x => !x) || pendingChecks.Any(x => !x))
238+
if (checks.Any(x => !x))
254239
return false;
255240

256-
if (baseSheetFile != null)
257-
{
258-
Sheet baseSheet;
259-
{
260-
using var f = File.OpenText(baseSheetFile);
261-
baseSheet = schemaDeserializer.Deserialize<Sheet>(f);
262-
}
263-
264-
bool[] baseChecks = [
265-
Validate<FieldNamesAndTypes>(baseSheet, sheet, cols, colDefs)
266-
];
267-
268-
if (!baseChecks.Any(x => x))
269-
return false;
270-
}
241+
// if (baseSheetFile != null)
242+
// {
243+
// Sheet baseSheet;
244+
// {
245+
// using var f = File.OpenText(baseSheetFile);
246+
// baseSheet = schemaDeserializer.Deserialize<Sheet>(f);
247+
// }
248+
249+
// bool[] baseChecks = [
250+
// Validate<FieldNamesAndTypes>(baseSheet, sheet, cols, colDefs)
251+
// ];
252+
253+
// if (!baseChecks.Any(x => x))
254+
// return false;
255+
// }
271256
return true;
272257
}
273258

274-
private static bool Validate<T>(Sheet sheet, List<ExcelColumnDefinition> cols, ColDefReader colDefs, bool pending = false) where T : IValidator<T>
259+
private static bool Validate<T>(Sheet sheet, ExcelColumnDefinition[] cols, ColDefReader colDefs) where T : IValidator<T>
275260
{
276261
try
277262
{
@@ -280,12 +265,13 @@ private static bool Validate<T>(Sheet sheet, List<ExcelColumnDefinition> cols, C
280265
}
281266
catch (Exception ex)
282267
{
283-
Log.AnnotatedError($"{typeof(T).Name}: {ex.Message}", new() { Title = $"Failed to validate{(pending ? " pending fields" : string.Empty)}", File = $"{sheet.Name}.yml" });
268+
Log.AnnotatedError($"{typeof(T).Name}: {ex.Message}", new() { Title = "Failed to validate", File = $"{sheet.Name}.yml" });
284269
return false;
285270
}
286271
}
287272

288-
private static bool Validate<T>(Sheet baseSheet, Sheet newSheet, List<ExcelColumnDefinition> cols, ColDefReader colDefs) where T : IBreakingValidator<T>
273+
[Obsolete("Pending fields are no longer used, but this may be useful in the future")]
274+
private static bool Validate<T>(Sheet baseSheet, Sheet newSheet, ExcelColumnDefinition[] cols, ColDefReader colDefs) where T : IBreakingValidator<T>
289275
{
290276
try
291277
{

‎apply.py‎

Lines changed: 2 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
1-
import sys
2-
31
from pathlib import Path
42
from ruamel.yaml import YAML
53
from ruamel.yaml.comments import CommentedMap
64
from ruamel.yaml.comments import CommentedSeq
75
from ruamel.yaml.compat import StringIO
86

9-
APPLY_PENDING = sys.argv[1] == 'true'
10-
117
class MyYAML(YAML):
128
def dump(self, data, stream=None, **kw):
139
inefficient = False
@@ -65,24 +61,6 @@ def apply_flow_style(data):
6561
data.ca.items['targets'] = comment
6662
return data
6763

68-
def apply_names(data):
69-
if APPLY_PENDING:
70-
if 'pendingName' in data:
71-
comment = data.ca.items.get('pendingName', None)
72-
data['name'] = data.pop('pendingName')
73-
if comment:
74-
data.ca.items['name'] = comment
75-
return data
76-
77-
def apply_changes(data):
78-
if APPLY_PENDING:
79-
if 'pendingFields' in data:
80-
comment = data.ca.items.get('pendingFields', None)
81-
data['fields'] = data.pop('pendingFields')
82-
if comment:
83-
data.ca.items['fields'] = comment
84-
return data
85-
8664
yaml = MyYAML()
8765
yaml.preserve_quotes = True
8866
yaml.width = 4096
@@ -92,28 +70,17 @@ def apply_changes(data):
9270
inp = path.read_text().strip()
9371
data = yaml.load(path)
9472

95-
data = order_keys(data, ['name', 'displayField', 'fields', 'pendingFields', 'relations'])
96-
data = apply_changes(data)
73+
data = order_keys(data, ['name', 'displayField', 'fields' 'relations'])
9774

9875
ret = []
9976
for fld in data['fields']:
100-
ret.append(recurse(fld, lambda f: order_keys(f, ['name', 'pendingName', 'comment', 'type', 'count', 'fields']), apply_flow_style, apply_names))
77+
ret.append(recurse(fld, lambda f: order_keys(f, ['name', 'comment', 'type', 'count', 'fields']), apply_flow_style, apply_names))
10178
if len(ret) != 0:
10279
comment = data.ca.items.get('fields', None)
10380
data['fields'] = ret
10481
if comment:
10582
data.ca.items['fields'] = comment
10683

107-
if 'pendingFields' in data:
108-
ret = []
109-
for fld in data['pendingFields']:
110-
ret.append(recurse(fld, lambda f: order_keys(f, ['name', 'pendingName', 'type', 'count', 'fields']), apply_flow_style, apply_names))
111-
if len(ret) != 0:
112-
comment = data.ca.items.get('pendingFields', None)
113-
data['pendingFields'] = ret
114-
if comment:
115-
data.ca.items['pendingFields'] = comment
116-
11784
output = yaml.dump(data).strip()
11885
if inp != output:
11986
print(path, 'changed')

‎schema.json‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,6 @@
2727
"$ref": "#/$defs/namedField"
2828
}
2929
},
30-
"pendingFields": {
31-
"description": "A list of new fields in the sheet, ordered by offset. When a new release is made, fields will be replaced with pendingFields.",
32-
"type": "array",
33-
"uniqueItems": true,
34-
"minItems": 1,
35-
"items": {
36-
"$ref": "#/$defs/namedField"
37-
}
38-
},
3930
"relations": {
4031
"$ref": "#/$defs/relations"
4132
}
@@ -267,11 +258,6 @@
267258
"description": "Name of the field",
268259
"type": "string",
269260
"pattern": "^\\w+$"
270-
},
271-
"pendingName": {
272-
"description": "New name of the field. If specified, the old name will be considered deprecated.",
273-
"type": "string",
274-
"pattern": "^\\w+$"
275261
}
276262
},
277263
"allOf": [

0 commit comments

Comments
 (0)