Allow DEA to be performed on a subset of the samples

Author: arteymix

gemma-cli/src/main/java/ubic/gemma/apps/DifferentialExpressionAnalysisCli.java

@@ -39,6 +39,7 @@
 import ubic.gemma.core.analysis.service.ExpressionDataFileService;
 import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysis;
 import ubic.gemma.model.common.auditAndSecurity.eventType.DifferentialExpressionAnalysisEvent;
+import ubic.gemma.model.expression.biomaterial.BioMaterial;
 import ubic.gemma.model.expression.experiment.*;
 import ubic.gemma.persistence.service.analysis.expression.diff.DifferentialExpressionAnalysisService;
 
@@ -80,21 +81,40 @@ private enum Mode {
         COMPLETE_FACTORS
     }
 
+    /**
+     * Mode for selecting which factors to include in the analysis.
+     */
+    private enum FactorSelectionMode {
+        /**
+         * Select factors based on a previous analysis.
+         */
+        REDO,
+        /**
+         * Select factors automatically.
+         */
+        AUTOMATIC,
+        /**
+         * Select factors manually. The values that are in {@link #factorIdentifiers} will be considered.
+         */
+        MANUAL
+    }
+
     /**
      * Specific analyses to redo.
      */
     @Nullable
-    private Collection<Long> analysisIds = null;
+    private Collection<Long> analysisIds;
 
     @Nullable
-    private Collection<Long> subsetIds = null;
+    private Collection<Long> subsetIds;
 
     /**
      * Indicate the type of analysis to perform.
      * <p>
      * The default is to detect it based on the dataset and the requested factors.
      */
-    private AnalysisType type = null;
+    @Nullable
+    private AnalysisType type;
 
     /**
      * Mode for selecting factors.
@@ -114,24 +134,33 @@ private enum Mode {
     private String subsetFactorIdentifier;
 
     /**
-     * Whether batch factors should be included (if they exist)
+     * Whether batch factors should be included (if they exist).
+     * <p>
+     * Defaults to true.
      */
-    private boolean ignoreBatch = true;
+    private boolean ignoreBatch;
 
     /**
      * Use moderated statistics.
      */
-    private boolean moderateStatistics = DifferentialExpressionAnalysisConfig.DEFAULT_MODERATE_STATISTICS;
+    private boolean moderateStatistics;
 
     /**
-     * Persist results to the database.
+     * Ignore failure of subset analyses.
+     * <p>
+     * The analysis will still fail if all subset analyses fail.
      */
-    private boolean persist = true;
+    private boolean ignoreFailingSubsets;
 
-    private boolean makeArchiveFiles = true;
-
-    private boolean ignoreFailingSubsets = false;
+    /**
+     * List of sample identifiers to include in the analysis.
+     * <p>
+     * Defaults to all samples.
+     */
+    @Nullable
+    private String[] sampleIdentifiers;
 
+    // data filtering options
     @Nullable
     private Integer filterMinNumberOfCellsPerSample;
     @Nullable
@@ -144,13 +173,22 @@ private enum Mode {
     @Nullable
     private Double filterMinVariance;
 
-    private DataFileOptionValue destination;
+    /**
+     * Persist results to the database.
+     */
+    private boolean persist ;
 
-    enum FactorSelectionMode {
-        REDO,
-        AUTOMATIC,
-        MANUAL
-    }
+    /**
+     * Create archive files.
+     */
+    private boolean makeArchiveFiles ;
+
+    /**
+     * Destination to use for archive files.
+     * <p>
+     * This is only applicable if {@link #makeArchiveFiles} is true.
+     */
+    private DataFileOptionValue destination;
 
     @Override
     public String getCommandName() {
@@ -197,6 +235,9 @@ protected void buildExperimentOptions( Options options ) {
                         + "If the experiment already has subsets for the factor, those will be reused. "
                         + "This is incompatible with -redo,--redo, -redoAnalysis,--redo-analysis or -redoSubset,--redo-subset." ).get() );
 
+        addSingleExperimentOption( options, Option.builder( "samples" ).longOpt( "samples" ).argName( "ID, name, accession" )
+                .desc( "ID, name or accession of samples to be included in the analysis. Defaults to all samples in the dataset being analyzed. Requires the " + formatOption( "nodb", "no-db" ) + " option to be set." ).get() );
+
         options.addOption( "usebatch", "use-batch-factor", false, "If a batch factor is available, use it. Otherwise, batch information can/will be ignored in the analysis. This is incompatible with " + formatOption( options, "factors" ) + ", -redo,--redo, -redoAnalysis,--redo-analysis and -redoSubset,--redo-subset." );
         options.addOption( "nobayes", "no-bayes", false, "Do not apply empirical-Bayes moderated statistics. Default is to use eBayes." );
         options.addOption( "ignoreFailingSubsets", "ignore-failing-subsets", false, "Ignore failing subsets and continue processing other subsets. Requires the " + formatOption( options, "subset" ) + " option to be set or -redo,--redo option with existing subset analyses." );
@@ -205,7 +246,7 @@ protected void buildExperimentOptions( Options options ) {
 
         // destination (db, standard location or custom directory)
         options.addOption( "nodb", "no-db", false, "Do not persist diff. ex. results to the database and instead save them to the current directory (or the location defined by " + formatOption( options, DataFileOptionsUtils.OUTPUT_DIR_OPTION ) + ")." );
-        options.addOption( "nofiles", "no-files", false, "Don't create archive files after analysis. Default is to make them. This is incompatible with " + formatOption( options, "nodb" ) + "." );
+        options.addOption( "nofiles", "no-files", false, "Don't create archive files after analysis. Default is to make them. This is incompatible with " + formatOption( options, "nodb" ) + " option to be set." );
 
         // redo mode
         options.addOption( "redo", "redo", false,
@@ -346,6 +387,7 @@ protected void processExperimentOptions( CommandLine commandLine ) throws ParseE
         // subset analysis can only be done in manual mode
         // note we add the given factor to the list of factors overall to make sure it is considered
         this.subsetFactorIdentifier = getOptionValue( commandLine, "subset", requires( allOf( toBeUnset( "redo" ), toBeUnset( "redoAnalysis" ), toBeUnset( "redoSubset" ) ) ) );
+        this.sampleIdentifiers = getOptionValues( commandLine, "samples", requires( toBeSet( "nodb" ) ) );
         // we can only force the use of a batch factor during automatic selection
         this.ignoreBatch = !hasOption( commandLine, "usebatch", requires( allOf( toBeUnset( "factors" ), toBeUnset( "redo" ), toBeUnset( "redoAnalysis" ), toBeUnset( "redoSubset" ) ) ) );
         this.moderateStatistics = !commandLine.hasOption( "nobayes" );
@@ -433,7 +475,18 @@ protected void processExpressionExperiment( ExpressionExperiment ee ) {
         config.setPersist( this.persist );
         config.setMakeArchiveFile( this.persist && this.makeArchiveFiles );
         config.setIgnoreFailingSubsets( this.ignoreFailingSubsets );
-        config.setUseWeights( super.eeService.isRNASeq( ee ) );
+        config.setUseWeights( eeService.isRNASeq( ee ) );
+
+        // sample selection
+        if ( this.sampleIdentifiers != null ) {
+            Set<BioMaterial> samplesToUse = new HashSet<>();
+            for ( String sampleId : sampleIdentifiers ) {
+                // allow searching in subsets, those are used in single-cell diff ex. analysis that use sub-samples for
+                // representing pseudo-bulks
+                samplesToUse.add( entityLocator.locateSample( ee, sampleId, true ) );
+            }
+            config.setSamplesToInclude( samplesToUse );
+        }
 
         // filtering
         config.setRepetitiveValuesFilterMode( this.filterMode );

gemma-cli/src/main/java/ubic/gemma/cli/completion/CompletionType.java

@@ -7,6 +7,7 @@ public enum CompletionType {
     DATASET_GROUP,
     EXTERNAL_DATABASE,
     DATASET,
+    SAMPLE,
     EXPERIMENTAL_FACTOR,
     /**
      * Complete experimental factors and interactions suitable for DEA.

gemma-cli/src/main/java/ubic/gemma/cli/util/EntityLocator.java

@@ -8,8 +8,10 @@
 import ubic.gemma.model.expression.bioAssayData.CellLevelCharacteristics;
 import ubic.gemma.model.expression.bioAssayData.CellTypeAssignment;
 import ubic.gemma.model.expression.bioAssayData.DataVector;
+import ubic.gemma.model.expression.biomaterial.BioMaterial;
 import ubic.gemma.model.expression.experiment.ExperimentalFactor;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
+import ubic.gemma.model.expression.experiment.ExpressionExperimentSubSet;
 import ubic.gemma.model.genome.Taxon;
 
 import java.util.Collection;
@@ -39,9 +41,35 @@ public interface EntityLocator {
 
     ExperimentalFactor locateExperimentalFactor( ExpressionExperiment expressionExperiment, String ctfName );
 
-    BioAssay locateBioAssay( ExpressionExperiment ee, String sampleId );
+    /**
+     * Locate an assay by its identifier.
+     *
+     * @param ee             dataset to lookup
+     * @param assayId        assay identifier to lookup
+     * @param includeSubSets whether to include assays that belong to subsets of the experiment. This is only relevant
+     *                       for {@link ExpressionExperimentSubSet} that "own" their assays instead of sharing them with
+     *                       the source experiment.
+     */
+    BioAssay locateBioAssay( ExpressionExperiment ee, String assayId, boolean includeSubSets );
 
+    /**
+     * Locate an assay by its identifier in a particular set of vectors.
+     *
+     * @param ee               dataset to lookup
+     * @param quantitationType quantitation type for the vectors
+     * @param sampleId         sample identifier to lookup
+     */
     BioAssay locateBioAssay( ExpressionExperiment ee, QuantitationType quantitationType, String sampleId );
 
+    /**
+     *
+     * @param ee             dataset to lookup
+     * @param sampleId       sample identifier to lookup
+     * @param includeSubSets whether to include samples associated to assays that belong to subsets of the experiment.
+     *                       This is only relevant for {@link ExpressionExperimentSubSet}
+     *                       that "own" their assays instead of sharing them with the source experiment.
+     */
+    BioMaterial locateSample( ExpressionExperiment ee, String sampleId, boolean includeSubSets );
+
     DifferentialExpressionAnalysis locateDiffExAnalysis( ExpressionExperiment ee, String analysisIdentifier );
 }

gemma-cli/src/main/java/ubic/gemma/cli/util/EntityLocatorImpl.java

@@ -2,6 +2,7 @@
 
 import lombok.extern.apachecommons.CommonsLog;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Strings;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
@@ -15,6 +16,7 @@
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssay.BioAssay;
 import ubic.gemma.model.expression.bioAssayData.*;
+import ubic.gemma.model.expression.biomaterial.BioMaterial;
 import ubic.gemma.model.expression.experiment.ExperimentalFactor;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.genome.Taxon;
@@ -268,7 +270,7 @@ public ExperimentalFactor locateExperimentalFactor( ExpressionExperiment express
         ExperimentalFactor factor;
         try {
             Long efId = Long.parseLong( identifier );
-            if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getId().equals( efId ) ) ) != null ) {
+            if ( ( factor = matchOneFactor( expressionExperiment, ef -> Objects.equals( ef.getId(), efId ) ) ) != null ) {
                 return factor;
             }
         } catch ( NumberFormatException e ) {
@@ -287,16 +289,16 @@ public ExperimentalFactor locateExperimentalFactor( ExpressionExperiment express
         }
 
         // match by category
-        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && StringUtils.equalsIgnoreCase( ef.getCategory().getCategory(), finalIdentifier ) ) ) != null ) {
+        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && Strings.CI.equals( ef.getCategory().getCategory(), finalIdentifier ) ) ) != null ) {
             return factor;
         }
-        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && StringUtils.equalsIgnoreCase( ef.getCategory().getCategoryUri(), finalIdentifier ) ) ) != null ) {
+        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && Strings.CI.equals( ef.getCategory().getCategoryUri(), finalIdentifier ) ) ) != null ) {
             return factor;
         }
-        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && StringUtils.equalsIgnoreCase( ef.getCategory().getValue(), finalIdentifier ) ) ) != null ) {
+        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && Strings.CI.equals( ef.getCategory().getValue(), finalIdentifier ) ) ) != null ) {
             return factor;
         }
-        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && StringUtils.equalsIgnoreCase( ef.getCategory().getValueUri(), finalIdentifier ) ) ) != null ) {
+        if ( ( factor = matchOneFactor( expressionExperiment, ef -> ef.getCategory() != null && Strings.CI.equals( ef.getCategory().getValueUri(), finalIdentifier ) ) ) != null ) {
             return factor;
         }
 
@@ -321,10 +323,9 @@ private ExperimentalFactor matchOneFactor( ExpressionExperiment ee, Predicate<Ex
     }
 
     @Override
-    public BioAssay locateBioAssay( ExpressionExperiment ee, String sampleId ) {
-        ee = eeService.thawLite( ee );
-        return requireNonNull( locateBioAssay( ee.getBioAssays(), sampleId ),
-                "Could not locate any assay matching '" + sampleId + "' in " + ee.getShortName() + "." + formatPossibleValues( ee.getBioAssays(), true ) );
+    public BioAssay locateBioAssay( ExpressionExperiment ee, String assayId, boolean includeSubSets ) {
+        return requireNonNull( locateBioAssay( eeService.getBioAssays( ee, includeSubSets ), assayId ),
+                "Could not locate any assay matching '" + assayId + "' in " + ee.getShortName() + "." + formatPossibleValues( ee.getBioAssays(), true ) );
     }
 
     @Override
@@ -342,47 +343,80 @@ public BioAssay locateBioAssay( ExpressionExperiment ee, QuantitationType qt, St
         throw new NullPointerException();
     }
 
-    @Override
-    public DifferentialExpressionAnalysis locateDiffExAnalysis( ExpressionExperiment ee, String analysisIdentifier ) {
-        return requireNonNull( differentialExpressionAnalysisService.findByExperimentAndAnalysisId( ee, true, Long.parseLong( analysisIdentifier ) ),
-                () -> String.format( "Could not locate an analysis matching '%s' in %s.%s",
-                        analysisIdentifier,
-                        ee.getShortName(),
-                        formatPossibleValues( differentialExpressionAnalysisService.findByExperiment( ee, true ), false ) ) );
-    }
-
     @Nullable
-    private BioAssay locateBioAssay( Collection<BioAssay> ee, String sampleId ) {
+    private BioAssay locateBioAssay( Collection<BioAssay> candidates, String sampleId ) {
         BioAssay ba;
         try {
             Long id = Long.parseLong( sampleId );
-            if ( ( ba = matchOneAssay( ee, ba2 -> ba2.getId().equals( id ) ) ) != null ) {
+            if ( ( ba = matchOneAssay( candidates, ba2 -> Objects.equals( ba2.getId(), id ) ) ) != null ) {
                 return ba;
             }
         } catch ( NumberFormatException e ) {
             // ignore
         }
-        if ( ( ba = matchOneAssay( ee, ba2 -> ba2.getShortName() != null && ba2.getShortName().equalsIgnoreCase( sampleId ) ) ) != null ) {
+        if ( ( ba = matchOneAssay( candidates, ba2 -> ba2.getShortName() != null && ba2.getShortName().equalsIgnoreCase( sampleId ) ) ) != null ) {
+            return ba;
+        }
+        if ( ( ba = matchOneAssay( candidates, ba2 -> ba2.getName().equalsIgnoreCase( sampleId ) ) ) != null ) {
             return ba;
         }
-        if ( ( ba = matchOneAssay( ee, ba2 -> ba2.getName().equalsIgnoreCase( sampleId ) ) ) != null ) {
+        if ( ( ba = matchOneAssay( candidates, ba2 -> ba2.getAccession() != null && ba2.getAccession().getAccession().equalsIgnoreCase( sampleId ) ) ) != null ) {
+            return ba;
+        }
+        return null;
+    }
+
+    private BioAssay matchOneAssay( Collection<BioAssay> candidates, Predicate<BioAssay> ba ) {
+        Set<BioAssay> bas = candidates.stream().filter( ba ).collect( Collectors.toSet() );
+        if ( bas.size() == 1 ) {
+            return bas.iterator().next();
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public BioMaterial locateSample( ExpressionExperiment ee, String sampleId, boolean includeSubSets ) {
+        return locateSample( eeService.getSamplesUsed( ee, includeSubSets ), sampleId );
+    }
+
+    private BioMaterial locateSample( Collection<BioMaterial> candidates, String sampleId ) {
+        BioMaterial ba;
+        try {
+            Long id = Long.parseLong( sampleId );
+            if ( ( ba = matchOneSample( candidates, ba2 -> Objects.equals( ba2.getId(), id ) ) ) != null ) {
+                return ba;
+            }
+        } catch ( NumberFormatException e ) {
+            // ignore
+        }
+        if ( ( ba = matchOneSample( candidates, ba2 -> ba2.getName().equalsIgnoreCase( sampleId ) ) ) != null ) {
             return ba;
         }
-        if ( ( ba = matchOneAssay( ee, ba2 -> ba2.getAccession() != null && ba2.getAccession().getAccession().equalsIgnoreCase( sampleId ) ) ) != null ) {
+        if ( ( ba = matchOneSample( candidates, ba2 -> ba2.getExternalAccession() != null && ba2.getExternalAccession().getAccession().equalsIgnoreCase( sampleId ) ) ) != null ) {
             return ba;
         }
         return null;
     }
 
-    private BioAssay matchOneAssay( Collection<BioAssay> bioAssays, Predicate<BioAssay> ba ) {
-        Set<BioAssay> bas = bioAssays.stream().filter( ba ).collect( Collectors.toSet() );
+    private BioMaterial matchOneSample( Collection<BioMaterial> candidates, Predicate<BioMaterial> ba ) {
+        Set<BioMaterial> bas = candidates.stream().filter( ba ).collect( Collectors.toSet() );
         if ( bas.size() == 1 ) {
             return bas.iterator().next();
         } else {
             return null;
         }
     }
 
+    @Override
+    public DifferentialExpressionAnalysis locateDiffExAnalysis( ExpressionExperiment ee, String analysisIdentifier ) {
+        return requireNonNull( differentialExpressionAnalysisService.findByExperimentAndAnalysisId( ee, true, Long.parseLong( analysisIdentifier ) ),
+                () -> String.format( "Could not locate an analysis matching '%s' in %s.%s",
+                        analysisIdentifier,
+                        ee.getShortName(),
+                        formatPossibleValues( differentialExpressionAnalysisService.findByExperiment( ee, true ), false ) ) );
+    }
+
     private String formatPossibleValues( Collection<? extends Identifiable> possibleValues, boolean allowAmbiguousIds ) {
         if ( possibleValues.isEmpty() ) {
             return "";