Skip to content

Commit e838c9f

Browse files
akarukappadathakarukappadath
authored
JupyterLab Extension Support (#714)
* fiab time * fixed test * right node and all * YAY * lab ext file * cleaning up PR * testing other input files * newest juplab * extracting in bash fine * trying again * all donegit add . * all done * more docs
1 parent 4af0894 commit e838c9f

File tree

11 files changed

+78
-12
lines changed

11 files changed

+78
-12
lines changed

‎automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoModelCopy.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,8 @@ case class ClusterRequest(labels: LabelMap = Map(),
8585

8686
case class UserJupyterExtensionConfig(nbExtensions: Map[String, String] = Map(),
8787
serverExtensions: Map[String, String] = Map(),
88-
combinedExtensions: Map[String, String] = Map())
88+
combinedExtensions: Map[String, String] = Map(),
89+
labExtensions: Map[String, String] = Map())
8990

9091

9192
case class ClusterError(errorMessage: String,

‎automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoTestUtils.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ trait LeonardoTestUtils extends WebBrowserSpec with Matchers with Eventually wit
9595
val dummyClusterSa = WorkbenchEmail("dummy-cluster")
9696
val dummyNotebookSa = WorkbenchEmail("dummy-notebook")
9797
val jupyterExtensions = clusterRequest.userJupyterExtensionConfig match {
98-
case Some(x) => x.nbExtensions ++ x.combinedExtensions ++ x.serverExtensions
98+
case Some(x) => x.nbExtensions ++ x.combinedExtensions ++ x.serverExtensions ++ x.labExtensions
9999
case None => Map()
100100
}
101101
val expected = clusterRequest.labels ++ DefaultLabels(clusterName, googleProject, creator, Some(dummyClusterSa), Some(dummyNotebookSa), clusterRequest.jupyterExtensionUri, clusterRequest.jupyterUserScriptUri).toMap ++ jupyterExtensions

‎automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/NotebookCustomizationSpec.scala‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,5 +78,19 @@ final class NotebookCustomizationSpec extends FreeSpec
7878
}
7979
}
8080
}
81+
82+
"should install user specified lab extensions" in {
83+
withProject { project => implicit token =>
84+
withNewCluster(project, request = ClusterRequest(userJupyterExtensionConfig = Some(UserJupyterExtensionConfig(labExtensions = Map("jupyterlab-toc" -> "@jupyterlab/toc"))))) { cluster =>
85+
withWebDriver { implicit driver =>
86+
withNewNotebook(cluster) { notebookPage =>
87+
val query = """!jupyter labextension list"""
88+
val result = notebookPage.executeCell(query).get
89+
result should include("@jupyterlab/toc")
90+
}
91+
}
92+
}
93+
}
94+
}
8195
}
8296
}

‎docker/jupyter/Dockerfile‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,9 @@ RUN apt-get update \
171171
python3.6 \
172172
python3.6-dev \
173173
python3-distutils \
174+
# for jupyterlab extensions
175+
nodejs \
176+
npm \
174177
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 100 \
175178
&& python3 get-pip.py \
176179
&& pip3 install tornado==4.5.3 \
@@ -184,7 +187,7 @@ RUN apt-get update \
184187
&& pip3 install pandas==0.23.4 \
185188
&& pip3 install seaborn==0.9.0 \
186189
&& pip3 install jupyter==1.0.0 \
187-
&& pip3 install jupyterlab==0.35.2 \
190+
&& pip3 install jupyterlab==0.35.4 \
188191
&& pip3 install python-lzo==1.12 \
189192
&& pip3 install google-api-core==1.5.0 \
190193
&& pip3 install google-cloud-bigquery==1.7.0 \
@@ -271,7 +274,8 @@ ADD custom/jupyter_delocalize.py $JUPYTER_HOME/custom/
271274
ADD custom/jupyter_localize_extension.py $JUPYTER_HOME/custom/
272275

273276
RUN chown -R $USER:users $JUPYTER_HOME \
274-
&& find $JUPYTER_HOME/scripts -name '*.sh' -type f | xargs chmod +x
277+
&& find $JUPYTER_HOME/scripts -name '*.sh' -type f | xargs chmod +x \
278+
&& chown -R $USER:users /usr/local/share/jupyter/lab
275279

276280
USER $USER
277281
WORKDIR $HOME

‎docker/jupyter/scripts/extension/jupyter_install_lab_extension.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
6+
if [ -n "$1" ]; then
7+
JUPYTER_EXTENSION=$1
8+
jupyter labextension install $JUPYTER_EXTENSION
9+
fi
10+

‎src/main/resources/jupyter/init-actions.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ if [[ "${ROLE}" == 'Master' ]]; then
8989
JUPYTER_SERVER_EXTENSIONS=$(jupyterServerExtensions)
9090
JUPYTER_NB_EXTENSIONS=$(jupyterNbExtensions)
9191
JUPYTER_COMBINED_EXTENSIONS=$(jupyterCombinedExtensions)
92+
JUPYTER_LAB_EXTENSIONS=$(jupyterLabExtensions)
9293
JUPYTER_CUSTOM_JS_URI=$(jupyterCustomJsUri)
9394
JUPYTER_GOOGLE_SIGN_IN_JS_URI=$(jupyterGoogleSignInJsUri)
9495
JUPYTER_USER_SCRIPT_URI=$(jupyterUserScriptUri)
@@ -247,6 +248,29 @@ if [[ "${ROLE}" == 'Master' ]]; then
247248
done
248249
fi
249250

251+
#Install lab extensions
252+
if [ ! -z "${JUPYTER_LAB_EXTENSIONS}" ] ; then
253+
for ext in ${JUPYTER_LAB_EXTENSIONS}
254+
do
255+
log 'Installing JupyterLab extension [$ext]...'
256+
pwd
257+
if [[ $ext == 'gs://'* ]]; then
258+
gsutil cp $ext /etc
259+
JUPYTER_EXTENSION_ARCHIVE=`basename $ext`
260+
docker cp /etc/${JUPYTER_EXTENSION_ARCHIVE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE}
261+
retry 3 docker exec ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_lab_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE}
262+
elif [[ $ext == 'http://'* || $ext == 'https://'* ]]; then
263+
JUPYTER_EXTENSION_FILE=`basename $ext`
264+
curl $ext -o /etc/${JUPYTER_EXTENSION_FILE}
265+
docker cp /etc/${JUPYTER_EXTENSION_FILE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE}
266+
retry 3 docker exec ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_lab_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE}
267+
268+
else
269+
retry 3 docker exec ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_lab_extension.sh $ext
270+
fi
271+
done
272+
fi
273+
250274

251275
retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/install_jupyter_contrib_nbextensions.sh
252276

‎src/main/resources/swagger/api-docs.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -972,6 +972,10 @@ definitions:
972972
description: |
973973
Optional, map of extension name and notebook plus server extension. The extension can either be a tar.gz file on google storage or a python package.
974974
Example, {"ext1":"gs://bucket/extension.tar.gz", "ext2":"python-package"}
975+
labExtensions:
976+
type: object
977+
description: |
978+
Optional, map of extension name and lab extension. The extension should be a verified jupyterlab extension that is uploaded to npm (list of public extensions here: https://github.com/search?utf8=%E2%9C%93&q=topic%3Ajupyterlab-extension&type=Repositories), a gzipped tarball made using 'npm pack', or a URL to a gzipped tarball made using 'npm pack'.
975979
976980
SubsystemStatus:
977981
description: status of a subsystem Leonardo depends on

‎src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/ExtensionComponent.scala‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,16 @@ trait ExtensionComponent extends LeoComponent {
4545
val nbExtensions = (recs.filter(_.extensionType == ExtensionType.NBExtension.toString) map { rec => rec.name -> rec.value }).toMap
4646
val serverExtensions = (recs.filter(_.extensionType == ExtensionType.ServerExtension.toString) map { rec => rec.name -> rec.value }).toMap
4747
val combinedExtensions = (recs.filter(_.extensionType == ExtensionType.CombinedExtension.toString) map { rec => rec.name -> rec.value }).toMap
48-
UserJupyterExtensionConfig(nbExtensions, serverExtensions, combinedExtensions)
48+
val labExtensions = (recs.filter(_.extensionType == ExtensionType.LabExtension.toString) map {rec => rec.name -> rec.value}).toMap
49+
UserJupyterExtensionConfig(nbExtensions, serverExtensions, combinedExtensions, labExtensions)
4950
}
5051
}
5152

5253
def marshallExtensions(clusterId:Long, userJupyterExtensionConfig: UserJupyterExtensionConfig): List[ExtensionRecord] = {
5354
((userJupyterExtensionConfig.nbExtensions map { case(key, value) => ExtensionRecord(clusterId, ExtensionType.NBExtension.toString, key, value)}) ++
5455
(userJupyterExtensionConfig.serverExtensions map { case(key, value) => ExtensionRecord(clusterId, ExtensionType.ServerExtension.toString, key, value)}) ++
55-
(userJupyterExtensionConfig.combinedExtensions map { case(key, value) => ExtensionRecord(clusterId, ExtensionType.CombinedExtension.toString, key, value)})).toList
56+
(userJupyterExtensionConfig.combinedExtensions map { case(key, value) => ExtensionRecord(clusterId, ExtensionType.CombinedExtension.toString, key, value)}) ++
57+
(userJupyterExtensionConfig.labExtensions map {case(key, value) => ExtensionRecord(clusterId, ExtensionType.LabExtension.toString, key, value)})).toList
5658
}
5759

5860
def unmarshallExtensions(extList: List[ExtensionRecord]): Option[UserJupyterExtensionConfig] = {
@@ -62,7 +64,8 @@ trait ExtensionComponent extends LeoComponent {
6264
val nbExtension = extList.filter(_.extensionType == ExtensionType.NBExtension.toString).map(x => x.name -> x.value).toMap
6365
val serverExtension = extList.filter(_.extensionType == ExtensionType.ServerExtension.toString).map(x => x.name -> x.value).toMap
6466
val combinedExtension = extList.filter(_.extensionType == ExtensionType.CombinedExtension.toString).map(x => x.name -> x.value).toMap
65-
Some(UserJupyterExtensionConfig(nbExtension, serverExtension, combinedExtension))
67+
val labExtension = extList.filter(_.extensionType == ExtensionType.LabExtension.toString).map(x => x.name -> x.value).toMap
68+
Some(UserJupyterExtensionConfig(nbExtension, serverExtension, combinedExtension, labExtension))
6669
}
6770
}
6871
}

‎src/main/scala/org/broadinstitute/dsde/workbench/leonardo/model/LeonardoModel.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ case class ClusterRequest(labels: Option[LabelMap] = Option(Map.empty),
3939

4040
case class UserJupyterExtensionConfig(nbExtensions: Map[String, String] = Map(),
4141
serverExtensions: Map[String, String] = Map(),
42-
combinedExtensions: Map[String, String] = Map())
42+
combinedExtensions: Map[String, String] = Map(),
43+
labExtensions: Map[String, String] = Map())
4344

4445

4546
// A resource that is required by a cluster
@@ -228,6 +229,7 @@ case class ClusterInitValues(googleProject: String,
228229
jupyterNbExtensions: String,
229230
jupyterCombinedExtensions: String,
230231
jupyterNotebookConfigUri: String,
232+
jupyterLabExtensions: String,
231233
defaultClientId: String
232234
){
233235
def toMap: Map[String, String] = this.getClass.getDeclaredFields.map(_.getName).zip(this.productIterator.to).toMap.mapValues(_.toString)}
@@ -262,6 +264,7 @@ object ClusterInitValues {
262264
clusterRequest.userJupyterExtensionConfig.map(x => x.nbExtensions.values.mkString(" ")).getOrElse(""),
263265
clusterRequest.userJupyterExtensionConfig.map(x => x.combinedExtensions.values.mkString(" ")).getOrElse(""),
264266
GcsPath(initBucketName, GcsObjectName(clusterResourcesConfig.jupyterNotebookConfigUri.value)).toUri,
267+
clusterRequest.userJupyterExtensionConfig.map(x => x.labExtensions.values.mkString(" ")).getOrElse(""),
265268
clusterRequest.defaultClientId.getOrElse("")
266269
)
267270
}
@@ -273,6 +276,7 @@ object ExtensionType extends Enum[ExtensionType] {
273276
case object NBExtension extends ExtensionType
274277
case object ServerExtension extends ExtensionType
275278
case object CombinedExtension extends ExtensionType
279+
case object LabExtension extends ExtensionType
276280
}
277281

278282
object LeonardoJsonSupport extends SprayJsonSupport with DefaultJsonProtocol {
@@ -295,7 +299,7 @@ object LeonardoJsonSupport extends SprayJsonSupport with DefaultJsonProtocol {
295299
}
296300
}
297301

298-
implicit val UserClusterExtensionConfigFormat = jsonFormat3(UserJupyterExtensionConfig.apply)
302+
implicit val UserClusterExtensionConfigFormat = jsonFormat4(UserJupyterExtensionConfig.apply)
299303

300304
implicit val ClusterRequestFormat = jsonFormat11(ClusterRequest)
301305

‎src/main/scala/org/broadinstitute/dsde/workbench/leonardo/service/LeonardoService.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -917,10 +917,12 @@ class LeonardoService(protected val dataprocConfig: DataprocConfig,
917917

918918
val combinedExtension = clusterRequest.userJupyterExtensionConfig.map(_.combinedExtensions).getOrElse(Map.empty)
919919

920+
val labExtension = clusterRequest.userJupyterExtensionConfig.map(_.labExtensions).getOrElse(Map.empty)
921+
920922
// combine default and given labels and add labels for extensions
921-
val allLabels = clusterRequest.labels.getOrElse(Map.empty) ++ defaultLabels ++ nbExtensions ++ serverExtensions ++ combinedExtension
923+
val allLabels = clusterRequest.labels.getOrElse(Map.empty) ++ defaultLabels ++ nbExtensions ++ serverExtensions ++ combinedExtension ++ labExtension
922924

923-
val updatedUserJupyterExtensionConfig = if(nbExtensions.isEmpty && serverExtensions.isEmpty && combinedExtension.isEmpty) None else Some(UserJupyterExtensionConfig(nbExtensions, serverExtensions, combinedExtension))
925+
val updatedUserJupyterExtensionConfig = if(nbExtensions.isEmpty && serverExtensions.isEmpty && combinedExtension.isEmpty && labExtension.isEmpty) None else Some(UserJupyterExtensionConfig(nbExtensions, serverExtensions, combinedExtension, labExtension))
924926

925927
// check the labels do not contain forbidden keys
926928
if (allLabels.contains(includeDeletedKey))

0 commit comments

Comments
 (0)