Added the reconstruct sub-command (#51)

* Add subcommand reconstruct

* Add a default reconstruct home folder to store compiled graph

* Copy and load enriched files to Blazegraph

* Implement code to query ASCT+B graph

* Implement code to load the graph

* Implement transformRecords to convert query result into ASCT+B table format

* Fix carriage return split

* Make sure the column order is correct

* Shorten concept IRI

* Fix the order between LABEL and ID

* Add header to the constructed table

* Fix references to remove duplicates and add PMID, if present

* Fix getting the reference ordering

* Normalized PubMed identifier

* Make ORCID field as string type

* Append only clonality and clone_id if they are present

* Implement reconstruct OMAP data

* Uncomment loadGraph

* Don't create any columns if the BM entry is empty

* Add at least a set of REF columns (e.g., REF/1, REF/1/ID) if no reference is present

* Implement reconstruct CTAnn data

* Implement reconstruct Ref-Organ data

* Get label from UBERON for accuracy

* Fix the data type for part_of_illustration property

* Implement reconstruct 2D FTU data

* Implement reconstruct Collection data

* Use Papaparse library to create the CSV content

* Remove quoteIfNeeded since we use Papaparse to handle quotes

* Move reconstruction artifacts to individual DO directories

* Cleanup reconstruction artifacts after generating the final output

Use the CLEANUP_ARTIFACTS flag to enable or disable this feature

* Convert query output from JSON to CSV using json2csv

* Change the output records to CSV format

* Fix missing general publications

* Remove unnecessary BIND(STR())

* Implement CSV parsing for ASCT+B DO outputs with Papaparse

* Remove the unnecessary BIND(STR()) for the rest of query files

* Implement CSV parsing for the rest DO outputs with Papaparse

* Add reference notes into the graph

* Add FTU columns into the graph

* Add FTU and reference notes to reconstruction process

* Fix sorting logic to handle numbers properly

* Fix malformed headers when reconstructing ASCT+B tables

* Add CT notes to the graph for use in reconstruction

* Remove unused folder to store the reconstruction artifacts

* Replace constant flag with --keep-artifacts CLI argument

* Extract queryGraph() into a reusable function

* Minor fix removing unnecessary quotes when reformatting date

* Add validation for reconstructed collection digital objects

* Fix array comparison to ignore item order

* Extract validation logging into its own module

* Implement validation technique through comparing CSV tables

* Reformat the warning log message

* Fix bug on locating the row number

* Reformat error log message

* Rename file

* Implement CTAnn validation that includes checks for table metadata section

* Add helper function to retrieve raw data name

* Undo comment out

* Implement OMAP validation

* Fix missing rationale for 'binds to antibody' relationships

* Add the Notes columns for AS and BM

* Make ASCT+B general publication query optional

* Modify ASCT+B record construction to include biomarker type for disambiguating duplicate entries

* Implement soft validation for the table header

* Enhance value comparison logic to handle array-like strings

* Cleanup the directory after validation

* Fix reference name and notes weren't captured properly

* Fix issue where protein presence is not retained in normalized form

* Add support for wildcard '*' to select columns for soft validation

* Ensure all columns from reconstructed CSV are present in the original CSV

* Skip row comparison if row sizes differ

* Rename variables

* Replace soft validation with per-cell comparison for unmatched rows

* Rename variables

Author: johardi

.gitignore

@@ -299,3 +299,6 @@ dist
 
 # VSCode settings
 .vscode
+
+# MacOS specific ignores
+.DS_Store

package-lock.json

@@ -33,6 +33,7 @@
     "commander": "^10.0.0",
     "glob": "^9.2.1",
     "js-yaml": "^4.1.0",
+    "jsonld-cli": "^0.2.0",
     "mime-types": "^2.1.35",
     "node-fetch": "^3.3.0",
     "nunjucks": "^3.2.4",
@@ -41,9 +42,9 @@
     "rdflib": "^2.2.34-1",
     "semver": "^7.5.3",
     "shelljs": "^0.8.5",
+    "sparqljson-parse": "^3.1.0",
     "tslib": "^2.5.0",
-    "validate-iri": "^1.0.1",
-    "jsonld-cli": "^0.2.0"
+    "validate-iri": "^1.0.1"
   },
   "devDependencies": {
     "eslint": "^8.35.0",

package.json

@@ -175,5 +175,6 @@ slots:
     description: >-
       Indicates the part of the illustration to which this node belongs to
       connect to the overall FTU structure.
+    range: EntityID
     annotations:
       owl: AnnotationProperty, AnnotationAssertion

schemas/src/digital-objects/2d-ftu.yaml

@@ -137,12 +137,8 @@ classes:
       - lipid_marker_list
       - metabolites_marker_list
       - proteoforms_marker_list
-      - references
-    slot_usage:
-      references:
-        range: string
-        annotations:
-          owl: AnnotationAssertion, AnnotationProperty
+      - ftu_list
+      - reference_list
 
   AnatomicalStructureRecord:
     title: Anatomical Structure Record
@@ -156,6 +152,7 @@ classes:
     slots:
       - ccf_pref_label
       - source_concept
+      - notes
       - record_number
       - order_number
 
@@ -171,6 +168,7 @@ classes:
     slots:
       - ccf_pref_label
       - source_concept
+      - notes
       - record_number
       - order_number
 
@@ -187,6 +185,39 @@ classes:
       - ccf_pref_label
       - ccf_biomarker_type
       - source_concept
+      - notes
+      - record_number
+      - order_number
+
+  FtuRecord:
+    title: Functional Tissue Unit Record
+    description: >-
+      A functional tissue unit (FTU) record represents a specific value in
+      the ASCT+B table, located in the "FTU" column, identified by its row
+      (record number) and column (order number).
+    mixins:
+      - Named
+      - Instance
+    slots:
+      - ccf_pref_label
+      - source_concept
+      - notes
+      - record_number
+      - order_number
+
+  ReferenceRecord:
+    title: Reference Record
+    description: >-
+      A reference record represents a specific value in the ASCT+B table, 
+      located in the "REF" column, identified by its row (record number) and 
+      column (order number).
+    mixins:
+      - Named
+      - Instance
+    slots:
+      - doi
+      - external_id
+      - notes
       - record_number
       - order_number
 
@@ -473,6 +504,42 @@ slots:
     slot_uri: ccf:proteoform_marker
     annotations:
       owl: AnnotationAssertion, AnnotationProperty
+  ftu_list:
+    title: functional tissue unit
+    description: List of FTUs referenced within ASCT+B records.
+    required: false
+    multivalued: true
+    inlined_as_list: true
+    range: FtuRecord
+    slot_uri: ccf:ftu_types
+    annotations:
+      owl: AnnotationAssertion, AnnotationProperty
+  reference_list:
+    title: reference list
+    description: List of references for the ASCT+B record.
+    required: false
+    multivalued: true
+    inlined_as_list: true
+    range: ReferenceRecord
+    slot_uri: ccf:reference
+    annotations:
+      owl: AnnotationAssertion, AnnotationProperty
+  doi:
+    title: doi
+    description: Digital Object Identifier for the reference record.
+    required: true
+    slot_uri: ccf:doi
+    range: uriorcurie
+    annotations:
+      owl: AnnotationAssertion, AnnotationProperty
+  external_id:
+    title: external id
+    description: External identifier for the reference record.
+    required: false
+    slot_uri: ccf:external_id
+    range: string
+    annotations:
+      owl: AnnotationAssertion, AnnotationProperty
   primary_cell_type:
     title: primary cell type
     description: The cell type in the cell marker descriptor.
@@ -503,3 +570,10 @@ slots:
     range: AsctbRecord
     annotations:
       owl: AnnotationAssertion, AnnotationProperty
+  notes:
+    title: notes
+    description: >-
+      Reference to the notes column in the ASCT+B table
+    slot_uri: ccf:notes
+    annotations:
+      owl: AnnotationAssertion, AnnotationProperty

schemas/src/digital-objects/asct-b.yaml

@@ -364,7 +364,6 @@ slots:
     title: author ORCID
     description: Reference to the author by the ORCID identifier.
     required: false
-    range: Named
     inlined: false
     multivalued: true
     annotations:

schemas/src/digital-objects/omap.yaml

@@ -9,6 +9,7 @@ import { newDraft } from './drafting/new-draft.js';
 import { enrich } from './enrichment/enrich.js';
 import { finalize } from './finalizing/finalize.js';
 import { validate } from './validation/validate.js';
+import { reconstruct } from './reconstruction/reconstruct.js';
 import { deployDoiXml } from './finalizing/misc-files.js';
 import { genAsctbCollectionJson } from './gen-asctb-collection-json.js';
 import { list } from './list.js';
@@ -118,6 +119,15 @@ program
     enrich(getContext(program, command, str));
   });
 
+program
+  .command('reconstruct')
+  .description('Reconstructs a given digital object from the enriched graph data')
+  .argument('<digital-object-path>', 'Path to the digital object relative to DO_HOME')
+  .option('--keep-artifacts', 'Keep intermediate files for debugging (default: cleanup artifacts)', false)
+  .action((str, _options, command) => {
+    reconstruct(getContext(program, command, str));
+  });
+
 program
   .command('build')
   .description('Given a Digital Object, checks for and runs normalization, enrichment, and packaging in one command.')

src/cli.js

@@ -135,6 +135,16 @@ export const objectFieldMap = {
   NOTE: 'notes',
 };
 
+export function getBiomarkerColumnName(biomarkerType) {
+  switch (biomarkerType) {
+    case "gene": return "BG";
+    case "protein": return "BP";
+    case "lipids": return "BL";
+    case "metabolites": return "BM";
+    case "proteoforms": return "BF";
+  }
+}
+
 export function createObject(name, structureType) {
   switch (structureType) {
     case 'REF':
@@ -146,11 +156,15 @@ export function createObject(name, structureType) {
 }
 
 export class Reference {
-  constructor(id) {
-    this.id = id;
+  constructor(name) {
+    this.name = name;
   }
   isValid() {
-    return !!this.id || !!this.doi || !!this.notes;
+    return !!this.id;
+  }
+  checkIsDoi(str) {
+    const doiRegex = /(10\.\d{4,9}\/[\w\-._;()/:]+)/i;
+    return doiRegex.test(str);
   }
 }
 
@@ -219,6 +233,15 @@ export class Row {
     this.cell_types = this.cell_types.filter((s) => s.isValid());
     this.ftu_types = this.ftu_types.filter((s) => s.isValid());
     this.references = this.references.filter((s) => s.isValid());
+    
+    // Remove duplicates based on 'id' property
+    const seenIds = new Set();
+    this.references = this.references.filter((s) => {
+      if (s.id && seenIds.has(s.id)) return false;
+      if (s.id) seenIds.add(s.id);
+      return true;
+    });
+    
     this.biomarkers_gene = this.biomarkers_gene.filter((s) => s.isValid());
     this.biomarkers_protein = this.biomarkers_protein.filter((s) => s.isValid());
     this.biomarkers_lipids = this.biomarkers_lipids.filter((s) => s.isValid());