Dockerization and updates

.devcontainer/Dockerfile

@@ -0,0 +1,11 @@
+FROM ia2/core:latest
+
+WORKDIR /tmp
+
+# Install dev packages
+COPY .devcontainer/requirements.txt .
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -r requirements.txt
+
+# Clean
+RUN rm -rf /tmp/*

.devcontainer/devcontainer.json

@@ -0,0 +1,44 @@
+{
+	"name": "ia2",
+	"initializeCommand": "make core-build",
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "ia2-devcontainer${localEnv:DEVCONTAINER_DEFAULT_DEVICE}",
+	"runServices": [
+		"ia2-devcontainer${localEnv:DEVCONTAINER_DEFAULT_DEVICE}"
+	],
+	"workspaceFolder": "/workspace",
+	"settings": {
+		"python.pythonPath": "/bin/python",
+		"python.languageServer": "Pylance",
+		"python.linting.enabled": true,
+		"python.linting.flake8Enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.linting.pycodestyleEnabled": false,
+		"python.formatting.blackPath": "/usr/local/bin/black",
+		"python.linting.flake8Path": ".flake8",
+		"python.linting.pylintPath": "/usr/local/bin/pylint",
+		"python.formatting.blackArgs": [
+			"--line-length=119"
+		],
+		"python.linting.pylintArgs": [
+			"--generate-members"
+		],
+		"editor.formatOnSave": true,
+		"python.formatting.provider": "black",
+		"files.trimFinalNewlines": true,
+		"files.trimTrailingWhitespace": true,
+		"files.watcherExclude": {
+			".git/**": true,
+			"./resources/**": true,
+			"./notebooks/**": true
+		}
+	},
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance",
+		"eamodio.gitlens",
+		"njpwerner.autodocstring",
+		"ms-azuretools.vscode-docker"
+	],
+	"postCreateCommand": "pip install -e /workspace/src/ia2 && git config --global --add safe.directory /workspace",
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3.10"
+services:
+    ia2-devcontainer:
+        network_mode: host
+        image: ia2-devcontainer
+        container_name: ia2-devcontainer
+        build:
+            context: ..
+            dockerfile: .devcontainer/Dockerfile
+        volumes:
+            - ..:/workspace:cached
+        # Please do not comment or remove this line!
+        command: /bin/sh -c "while sleep 1000; do :; done"
+

.devcontainer/requirements.txt

@@ -0,0 +1,5 @@
+# code style
+
+black==22.8.0
+flake8==5.0.4
+pylint==2.15.2

.dockerignore

@@ -0,0 +1,10 @@
+.vscode
+.python-version
+Pipfile.lock
+**/__pycache__
+**/*.ipynb_checkpoints
+**/*.egg-info
+
+.git/
+resources/
+notebooks/

.gitignore

@@ -1,27 +1,35 @@
 # Text Editors
 .vscode
+.python-version
+Pipfile.lock
+**/*.ipynb_checkpoints
+**/*.egg-info
+**/*.virtual_documents
+
+**/*.csv
+**/*.pkl
 
 # Training Configuration
 train_config.json
 
 # Spacy Models
-models/*
-!models/.gitkeep
+src/ia2/ia2/models/*
+!src/ia2/ia2/models/.gitkeep
 
 # Logs
-logs/*
-!logs/.gitkeep
+src/ia2/ia2/logs/*
+!src/ia2/ia2/logs/.gitkeep
 
 # Training History
-history/*
-!history/.gitkeep
+src/ia2/ia2/history/*
+!src/ia2/ia2/history/.gitkeep
 
 # Datasets
-data/*
-!data/.gitkeep
+resources/dataset/*
+!resources/dataset/.gitkeep
 
 # Misc
-__pycache__
+**/__pycache__
 
 # Logs
 logs

Makefile

@@ -18,3 +18,26 @@ lint:
 #format: @ Run black code formatter
 format:
 	python -m black --config ./pyproject.toml .
+
+
+include .env
+export $(shell sed 's/=.*//' .env)
+
+core-build:
+	docker-compose build ia2-core
+
+core-run:
+	docker-compose run ia2-core
+
+jupyter-build: core-build
+	docker-compose build ia2-jupyter
+
+jupyter-run: jupyter-build
+	docker-compose up ia2-jupyter-gpu
+
+jupyter-run-cpu: jupyter-build
+	docker-compose up ia2-jupyter-cpu
+
+core-test-cpu-all: export TEST_COMMAND=$(BASE_TEST_COMMAND)
+core-test-cpu-all: core-build
+	docker-compose run ia2-core-test-cpu

build/core/Dockerfile

@@ -0,0 +1,70 @@
+FROM nvcr.io/nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+
+ARG PIP_VERSION
+ARG PYTHON_VERSION
+ARG CUDA_VERSION
+ARG TORCH_VERSION
+
+# OS packages
+RUN sed -i 's/# deb-src/deb-src/' /etc/apt/sources.list \
+    && apt-get update --fix-missing; exit 0
+
+RUN apt-get install -y --no-install-recommends \
+    software-properties-common \
+    && add-apt-repository ppa:deadsnakes/ppa
+
+RUN apt-get install -y --no-install-recommends \
+    curl \
+    wget \
+    git-core \
+    htop \
+    unzip \
+    zsh \
+    vim \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python${PYTHON_VERSION}-distutils \
+    libtool \
+    libleptonica-dev \
+    libltdl-dev \
+    pkg-config \
+    libgl1-mesa-glx \
+    python3-setuptools \
+    && apt-get autoremove -y \
+    && apt-get clean \
+    && rm -rf /var/tmp/* /var/lib/apt/lists/*
+
+# Set default python version
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 \
+    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1
+
+WORKDIR /tmp
+
+# Install pip
+RUN wget https://github.com/pypa/pip/archive/refs/tags/${PIP_VERSION}.zip \
+    && unzip ${PIP_VERSION}.zip \
+    && cd pip-${PIP_VERSION} \
+    && python setup.py install
+
+# Install pytorch
+RUN --mount=type=cache,target=/root/.cache \
+    pip install torch==${TORCH_VERSION}+${CUDA_VERSION} \
+    -f https://download.pytorch.org/whl/cu113/torch_stable.html
+
+#Install Spacy
+RUN pip install -U pip setuptools wheel \
+    pip install -U spacy[${CUDA_VERSION}] \
+    pip install spacy-transformers \
+    pip install cupy-wheel
+
+# install requirements
+COPY ./src/ia2 /tmp/ia2
+RUN --mount=type=cache,target=/root/.cache \
+    pip install ./ia2
+
+RUN rm -rf /tmp/*
+
+WORKDIR /root

build/jupyter/Dockerfile

@@ -0,0 +1,30 @@
+ARG CORE_IMAGE
+
+FROM ${CORE_IMAGE}
+
+WORKDIR /tmp
+
+# Jupyter terminal with zsh
+RUN sh -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
+ENV SHELL=/usr/bin/zsh
+RUN chsh root -s $SHELL
+
+# Therminal dark theme
+RUN mkdir -p ~/.jupyter/lab/user-settings/@jupyterlab/terminal-extension \
+    && echo '{"theme": "dark"}' > ~/.jupyter/lab/user-settings/@jupyterlab/terminal-extension/plugin.jupyterlab-settings
+
+# Execution time & max number of outputs
+RUN mkdir -p ~/.jupyter/lab/user-settings/@jupyterlab/notebook-extension \
+    && echo '{"recordTiming": true, "maxNumberOutputs": 10000}' > ~/.jupyter/lab/user-settings/@jupyterlab/notebook-extension/tracker.jupyterlab-settings
+
+COPY ./build/jupyter/requirements.txt .
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -r requirements.txt
+
+# Clean
+RUN rm -rf /tmp/*
+
+WORKDIR /notebooks
+
+COPY ./build/jupyter/entrypoint.sh /usr/bin/entrypoint.sh
+ENTRYPOINT sh /usr/bin/entrypoint.sh

build/jupyter/entrypoint.sh

@@ -0,0 +1,4 @@
+# reinstall local packages with no deps & edition mode
+pip install --no-deps -e /src/ia2
+
+jupyter-lab --ip=0.0.0.0 --allow-root --no-browser

build/jupyter/requirements.txt

@@ -0,0 +1,3 @@
+jupyterlab==3.4.6
+ipywidgets==8.0.2
+jupyterlab_execute_time==2.1.0

docker-compose.yml

@@ -0,0 +1,70 @@
+version: "3.10"
+services:
+    ia2-core:
+        image: ${CORE_IMAGE}
+        container_name: ia2-core
+        build:
+            context: .
+            dockerfile: ./build/core/Dockerfile
+            args:
+                - PYTHON_VERSION=${PYTHON_VERSION}
+                - PIP_VERSION=${PIP_VERSION}
+                - CUDA_VERSION=${CUDA_VERSION}
+                - TORCH_VERSION=${TORCH_VERSION}
+                - TEST_MODEL_FILE=${TEST_MODEL_FILE}
+        volumes:
+            - $PWD/${DATASETS_VOLUME_MOUNT}
+            - $PWD/${RESOURCES_VOLUME_MOUNT}
+        env_file:
+            - .env
+
+    ia2-jupyter:
+        image: ${JUPYTER_IMAGE}
+        container_name: ia2-jupyter
+        build:
+            context: .
+            dockerfile: ./build/jupyter/Dockerfile
+            args:
+                - CORE_IMAGE=${CORE_IMAGE}
+
+    ia2-jupyter-gpu:
+        network_mode: host
+        image: ${JUPYTER_IMAGE}
+        container_name: ia2-jupyter-gpu
+        deploy:
+            resources:
+                reservations:
+                    devices:
+                        - capabilities: [ gpu ]
+        shm_size: '8gb'
+        volumes:
+            - $PWD/${SRC_VOLUME_MOUNT}
+            - $PWD/${NOTEBOOKS_VOLUME_MOUNT}
+            - $PWD/${RESOURCES_VOLUME_MOUNT}
+        env_file:
+            - .env
+
+    ia2-jupyter-cpu:
+        network_mode: host
+        image: ${JUPYTER_IMAGE}
+        container_name: ia2-jupyter-cpu
+        shm_size: '8gb'
+        volumes:
+            - $PWD/${SRC_VOLUME_MOUNT}
+            - $PWD/${NOTEBOOKS_VOLUME_MOUNT}
+            - $PWD/${RESOURCES_VOLUME_MOUNT}
+        env_file:
+            - .env
+
+    ia2-core-test-cpu:
+        network_mode: host
+        image: ${CORE_IMAGE}
+        command: sh -c " cd .. && ${TEST_COMMAND}"
+        container_name: ia2-core-test-cpu
+        shm_size: '8gb'
+        volumes:
+            - $PWD/${TEST_VOLUME_MOUNT}
+            - $PWD/${SRC_VOLUME_MOUNT}
+            - $PWD/${RESOURCES_VOLUME_MOUNT}
+        env_file:
+            - .env

src/ia2/setup.py

@@ -0,0 +1,23 @@
+from setuptools import find_packages, setup
+
+setup(
+    name="ia2",
+    packages=find_packages(),
+    version="1.1.0",
+    description="Command Line Interface for IA² models development, training and deployment.",
+    author="IA² - Instituciones Abiertas",
+    author_email="info@ia2.coop",
+    url="https://github.com/instituciones-abiertas/ia2-cli",
+    install_requires=[
+        "jsonschema==4.5.1",
+        "pre-commit==2.20.0",
+        "fire==0.4.0",
+        "more-itertools==8.13.0",
+        "pandas==1.4.2",
+        "matplotlib==3.5.2",
+        "seaborn==0.11.2",
+    ],
+    package_data={"": ["*.yml", "*.yaml"]},
+    include_package_data=True,
+    classifiers=["Programming Language :: Python :: 3"],
+)