added make-config script and fixed curl

Author: bothness

.github/workflows/update.yml

@@ -20,14 +20,14 @@ jobs:
         uses: actions/checkout@v2
       - name: Fetch data
         run: |-
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=localities" -o "raw-data/localities.csv"
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=layers" -o "raw-data/layers.csv"
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=sheets" -o "raw-data/sheets.csv"
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=authors" -o "raw-data/authors.csv"
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=sources" -o "raw-data/sources.csv"
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=statuses" -o "raw-data/statuses.csv"
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=groups" -o "raw-data/groups.csv"
-          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=translations" -o "raw-data/translations.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=localities" -o "raw-data/localities.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=layers" -o "raw-data/layers.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=sheets" -o "raw-data/sheets.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=authors" -o "raw-data/authors.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=sources" -o "raw-data/sources.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=statuses" -o "raw-data/statuses.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=groups" -o "raw-data/groups.csv"
+          curl "https://docs.google.com/spreadsheets/d/1e7A-maXjmzd0lRcpZpeqjOmfRTZUZMOaxSYQaz_sr8I/gviz/tq?tqx=out:csv&sheet=translations" -o "raw-data/translations.csv"
       - name: Process data
         run: |-
           git diff --quiet raw-data/ || deno run --allow-read --allow-write --unstable scripts/make-places.js

scripts/config.js

@@ -2,4 +2,5 @@ export const locs_path = "raw-data/localities.csv";
 export const places_path = "data/places.json";
 export const redirect_path = "data/redirects.json";
 export const layers_path = "data/layers.json";
-export const sheets_path = "data/sheets.json";
+export const sheets_path = "data/sheets.json";
+export const config_path = "data/config.json";

scripts/make-config.js

@@ -0,0 +1,37 @@
+import { readTXT, writeJSON } from 'https://deno.land/x/flat@0.0.15/mod.ts';
+import { csvParse, autoType } from "https://unpkg.com/d3-dsv@3.0.1/src/index.js";
+import { config_path } from "./config.js";
+
+const keys = ["authors", "groups", "layers", "sources", "statuses", "translations"];
+const parseForBoolean = (val) => val === "TRUE" ? true : val === "FALSE" ? false : val;
+
+const config = {};
+
+for (const key of keys) {
+  const path = `raw-data/${key}.csv`;
+  console.log(`Reading ${path}`);
+  const data = csvParse(await readTXT(path), autoType);
+
+  config[key] = data.map(d => {
+		const row = {};
+		for (const col of data.columns) {
+			const arr = col.match(/(?<=\[)\d*(?=\])/);
+			if (Array.isArray(arr)) {
+				const key = col.split("[")[0];
+				if (arr[0] === "") row[key] = d[col].split(", ");
+				else {
+					if (!row[key]) row[key] = [];
+					row[key][arr[0]] = parseForBoolean(d[col]);
+				}
+			} else row[col] = parseForBoolean(d[col]);
+		}
+		const newCols = Object.keys(row);
+		for (const col of newCols) {
+			if (Array.isArray(row[col]) && row[col].every(d => !d)) row[col] = null;
+		}
+		return row;
+	});
+}
+
+writeJSON(config_path, config);
+console.log(`Wrote ${config_path}`);