Merge pull request #213 from open-runtimes/refactor-extract-imagepuller

refactor: extract imagepuller

Author: loks0n

.env

@@ -1,6 +1,5 @@
 OPR_EXECUTOR_ENV=development
-OPR_EXECUTOR_IMAGE_PULL=enabled
-OPR_EXECUTOR_RUNTIMES=php-8.1,dart-2.18,deno-1.24,node-18.0,python-3.10,ruby-3.1,cpp-17,dotnet-6.0
+OPR_EXECUTOR_IMAGES=openruntimes/php:v5-8.1,openruntimes/php:v2-8.1,openruntimes/dart:v5-2.18,openruntimes/deno:v5-1.24,openruntimes/deno:v2-1.24,openruntimes/node:v5-18.0,openruntimes/node:v2-18.0,openruntimes/python:v5-3.10,openruntimes/python:v2-3.10,openruntimes/ruby:v5-3.1,openruntimes/ruby:v2-3.1,openruntimes/cpp:v5-17,openruntimes/cpp:v2-17,openruntimes/dotnet:v5-6.0,openruntimes/dotnet:v2-6.0
 OPR_EXECUTOR_CONNECTION_STORAGE=local://localhost
 OPR_EXECUTOR_INACTIVE_THRESHOLD=60
 OPR_EXECUTOR_MAINTENANCE_INTERVAL=60

README.md

@@ -49,7 +49,7 @@ services:
       - ./functions:/storage/functions:rw
     environment:
       - OPR_EXECUTOR_ENV
-      - OPR_EXECUTOR_RUNTIMES
+      - OPR_EXECUTOR_IMAGES
       - OPR_EXECUTOR_CONNECTION_STORAGE
       - OPR_EXECUTOR_INACTIVE_THRESHOLD
       - OPR_EXECUTOR_MAINTENANCE_INTERVAL
@@ -79,9 +79,8 @@ volumes:
 
 ```
 OPR_EXECUTOR_ENV=development
-OPR_EXECUTOR_RUNTIMES=php-8.0
+OPR_EXECUTOR_IMAGES=openruntimes/php:v5-8.3
 OPR_EXECUTOR_CONNECTION_STORAGE=file://localhost
-OPR_EXECUTOR_IMAGE_PULL=enabled
 OPR_EXECUTOR_INACTIVE_THRESHOLD=60
 OPR_EXECUTOR_MAINTENANCE_INTERVAL=60
 OPR_EXECUTOR_NETWORK=openruntimes-runtimes
@@ -192,7 +191,7 @@ docker compose down
 | Variable name                    | Description                                                                                                                                   |
 |------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
 | OPR_EXECUTOR_ENV                 | Environment mode of the executor, ex. `development`                                                                                           |
-| OPR_EXECUTOR_RUNTIMES            | Comma-separated list of supported runtimes `(ex: php-8.1,dart-2.18,deno-1.24,..)`. These runtimes should be available as container images.    |
+| OPR_EXECUTOR_IMAGES            | Comma-separated list of supported images `(ex: openruntimes/php:v5-8.1,openruntimes/php:v2-8.1,..)`. |
 | OPR_EXECUTOR_CONNECTION_STORAGE  | DSN string that represents a connection to your storage device, ex: `file://localhost` for local storage                                      |
 | OPR_EXECUTOR_INACTIVE_THRESHOLD   | Threshold time (in seconds) for detecting inactive runtimes, ex: `60`                                                                         |
 | OPR_EXECUTOR_MAINTENANCE_INTERVAL| Interval (in seconds) at which the Executor performs maintenance tasks, ex: `60`                                                              |
@@ -205,7 +204,6 @@ docker compose down
 | OPR_EXECUTOR_RUNTIME_VERSIONS    | Version tag for runtime environments, ex: `v5`                                                                                                |
 | OPR_EXECUTOR_RETRY_ATTEMPTS      | Number of retry attempts for failed executions, ex: `5`                                                                                       |
 | OPR_EXECUTOR_RETRY_DELAY_MS      | Delay (in milliseconds) between retry attempts, ex: `500`                                                                                    |
-| OPR_EXECUTOR_IMAGE_PULL      | Pull open runtimes images before executor starts. Takes `disabled` and `enabled` |
 
 ## Contributing
 

app/http.php

@@ -8,9 +8,11 @@
 require_once __DIR__ . '/controllers.php';
 
 use OpenRuntimes\Executor\Runner\Docker;
+use OpenRuntimes\Executor\Runner\ImagePuller;
 use OpenRuntimes\Executor\Runner\Maintenance;
 use OpenRuntimes\Executor\Runner\Repository\Runtimes;
 use OpenRuntimes\Executor\Runner\NetworkManager;
+use Swoole\Process;
 use Swoole\Runtime;
 use Utopia\Console;
 use Utopia\Http\Http;
@@ -57,6 +59,10 @@
     $selfContainer = $orchestration->list(['name' => $hostname])[0] ?? throw new \RuntimeException('Own container not found');
     $networkManager->connectAll($selfContainer);
 
+    /* Pull images */
+    $imagePuller = new ImagePuller($orchestration);
+    $imagePuller->pull(explode(',', System::getEnv('OPR_EXECUTOR_IMAGES') ?: ''));
+
     /* Start maintenance task */
     $maintenance = new Maintenance($orchestration, $runtimes);
     $maintenance->start(
@@ -71,6 +77,12 @@
     $server = new Server('0.0.0.0', '80', $settings);
     $http = new Http($server, 'UTC');
 
+    Process::signal(SIGTERM, function () use ($maintenance, $runner) {
+        // This doesn't actually work. We need to fix utopia-php/http@0.34.x
+        $maintenance->stop();
+        $runner->cleanUp();
+    });
+
     Console::success('Executor is ready.');
 
     $http->start();

composer.json

@@ -20,7 +20,6 @@
     "ext-curl": "*",
     "ext-json": "*",
     "ext-swoole": "*",
-    "appwrite/php-runtimes": "0.19.*",
     "utopia-php/config": "^0.2.2",
     "utopia-php/console": "0.0.*",
     "utopia-php/dsn": "0.2.*",

composer.lock

@@ -5,10 +5,10 @@
 
 x-logging: &x-logging
   logging:
-    driver: 'json-file'
+    driver: "json-file"
     options:
-      max-file: '5'
-      max-size: '10m'
+      max-file: "5"
+      max-size: "10m"
 
 services:
   openruntimes-executor:
@@ -40,7 +40,7 @@ services:
       - ./tests/resources/sites:/storage/sites:rw
     environment:
       - OPR_EXECUTOR_ENV
-      - OPR_EXECUTOR_RUNTIMES
+      - OPR_EXECUTOR_IMAGES
       - OPR_EXECUTOR_CONNECTION_STORAGE
       - OPR_EXECUTOR_INACTIVE_THRESHOLD
       - OPR_EXECUTOR_MAINTENANCE_INTERVAL

docker-compose.yml

@@ -3,12 +3,10 @@
 namespace OpenRuntimes\Executor\Runner;
 
 use OpenRuntimes\Executor\Logs;
-use Appwrite\Runtimes\Runtimes as AppwriteRuntimes;
 use OpenRuntimes\Executor\Exception;
 use OpenRuntimes\Executor\Runner\Repository\Runtimes;
 use OpenRuntimes\Executor\StorageFactory;
 use OpenRuntimes\Executor\Validator\TCP;
-use Swoole\Process;
 use Swoole\Timer;
 use Throwable;
 use Utopia\Console;
@@ -34,52 +32,6 @@ public function __construct(
         private readonly Runtimes $runtimes,
         private readonly NetworkManager $networkManager
     ) {
-        $this->init();
-    }
-
-    /**
-     * @return void
-     * @throws \Utopia\Http\Exception
-     */
-    private function init(): void
-    {
-        /**
-         * Warmup: make sure images are ready to run fast 🚀
-         */
-        $allowList = empty(System::getEnv('OPR_EXECUTOR_RUNTIMES')) ? [] : \explode(',', System::getEnv('OPR_EXECUTOR_RUNTIMES'));
-
-        if (System::getEnv('OPR_EXECUTOR_IMAGE_PULL', 'enabled') === 'disabled') {
-            // Useful to prevent auto-pulling from remote when testing local images
-            Console::info("Skipping image pulling");
-        } else {
-            $runtimeVersions = \explode(',', System::getEnv('OPR_EXECUTOR_RUNTIME_VERSIONS', 'v5') ?? 'v5');
-            foreach ($runtimeVersions as $runtimeVersion) {
-                Console::success("Pulling $runtimeVersion images...");
-                $images = new AppwriteRuntimes($runtimeVersion); // TODO: @Meldiron Make part of open runtimes
-                $images = $images->getAll(true, $allowList);
-                $callables = [];
-                foreach ($images as $image) {
-                    $callables[] = function () use ($image) {
-                        Console::log('Warming up ' . $image['name'] . ' ' . $image['version'] . ' environment...');
-                        $response = $this->orchestration->pull($image['image']);
-                        if ($response) {
-                            Console::info("Successfully Warmed up {$image['name']} {$image['version']}!");
-                        } else {
-                            Console::warning("Failed to Warmup {$image['name']} {$image['version']}!");
-                        }
-                    };
-                }
-
-                batch($callables);
-            }
-        }
-
-        Console::success("Image pulling finished.");
-
-        Process::signal(SIGINT, fn () => $this->cleanUp());
-        Process::signal(SIGQUIT, fn () => $this->cleanUp());
-        Process::signal(SIGKILL, fn () => $this->cleanUp());
-        Process::signal(SIGTERM, fn () => $this->cleanUp());
     }
 
     /**
@@ -1107,7 +1059,7 @@ public function createExecution(
     /**
      * @return void
      */
-    private function cleanUp(): void
+    public function cleanUp(): void
     {
         Console::log('Cleaning up containers and networks...');
 

src/Executor/Runner/Docker.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace OpenRuntimes\Executor\Runner;
+
+use Utopia\Console;
+use Utopia\Orchestration\Orchestration;
+
+use function Swoole\Coroutine\batch;
+
+readonly class ImagePuller
+{
+    public function __construct(private Orchestration $orchestration)
+    {
+    }
+
+    /**
+     * Pulls images from the registry.
+     *
+     * @param list<string> $images
+     */
+    public function pull(array $images): void
+    {
+        if (empty($images)) {
+            Console::log('[ImagePuller] No images to pull.');
+            return;
+        }
+
+        $jobs = array_map(fn ($image) => function () use ($image) {
+            if (!$this->orchestration->pull($image)) {
+                Console::error("[ImagePuller] Failed to pull image $image");
+                return;
+            }
+
+            return true;
+        }, $images);
+
+        go(function () use ($jobs) {
+            $results = batch($jobs);
+            $success = \count(array_filter($results));
+
+            Console::info("[ImagePuller] Pulled $success/". \count($jobs) . " images.");
+        });
+
+        Console::info('[ImagePuller] Started pulling images.');
+    }
+}