Merge pull request #845 from 10up/feature/localhost

Add Ollama as a Provider, allowing some Features to work with locally hosted LLMs

Author: dkotter

.github/workflows/cypress.yml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         core:
           - {name: 'WP latest', version: 'latest'}
-          - {name: 'WP minimum', version: 'WordPress/WordPress#6.5'}
+          - {name: 'WP minimum', version: 'WordPress/WordPress#6.6'}
           - {name: 'WP trunk', version: 'WordPress/WordPress#master'}
 
     steps:

README.md

@@ -16,20 +16,22 @@ Tap into leading cloud-based services like [OpenAI](https://openai.com/), [Micro
 
 ## Features
 
-* Generate a summary of post content and store it as an excerpt using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service) or [Google's Gemini API](https://ai.google.dev/docs/gemini_api_overview)
-* Generate key takeaways from post content and render at the top of a post using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat) or [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
-* Generate titles from post content using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service) or [Google's Gemini API](https://ai.google.dev/docs/gemini_api_overview)
-* Expand or condense text content using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service) or [Google's Gemini API](https://ai.google.dev/docs/gemini_api_overview)
+* Generate a summary of post content and store it as an excerpt using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service), [Google's Gemini API](https://ai.google.dev/docs/gemini_api_overview), [xAI's Grok](https://x.ai/) or locally using [Ollama](https://ollama.com/)
+* Generate key takeaways from post content and render at the top of a post using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service) or locally using [Ollama](https://ollama.com/)
+* Generate titles from post content using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service), [Google's Gemini API](https://ai.google.dev/docs/gemini_api_overview), [xAI's Grok](https://x.ai/) or locally using [Ollama](https://ollama.com/)
+* Expand or condense text content using [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service), [Google's Gemini API](https://ai.google.dev/docs/gemini_api_overview), [xAI's Grok](https://x.ai/) or locally using [Ollama](https://ollama.com/)
 * Generate new images on demand to use in-content or as a featured image using [OpenAI's DALL·E 3 API](https://platform.openai.com/docs/guides/images)
 * Generate transcripts of audio files using [OpenAI's Whisper API](https://platform.openai.com/docs/guides/speech-to-text)
 * Moderate incoming comments for sensitive content using [OpenAI's Moderation API](https://platform.openai.com/docs/guides/moderation)
 * Convert text content into audio and output a "read-to-me" feature on the front-end to play this audio using [Microsoft Azure's Text to Speech API](https://learn.microsoft.com/en-us/azure/cognitive-services/speech-service/text-to-speech), [Amazon Polly](https://aws.amazon.com/polly/) or [OpenAI's Text to Speech API](https://platform.openai.com/docs/guides/text-to-speech)
-* Classify post content using [IBM Watson's Natural Language Understanding API](https://www.ibm.com/watson/services/natural-language-understanding/), [OpenAI's Embedding API](https://platform.openai.com/docs/guides/embeddings) or [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
+* Classify post content using [IBM Watson's Natural Language Understanding API](https://www.ibm.com/watson/services/natural-language-understanding/), [OpenAI's Embedding API](https://platform.openai.com/docs/guides/embeddings), [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service) or locally using [Ollama](https://ollama.com/)
 * Create a smart 404 page that has a recommended results section that suggests relevant content to the user based on the page URL they were trying to access using either [OpenAI's Embedding API](https://platform.openai.com/docs/guides/embeddings) or [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service) in combination with [ElasticPress](https://github.com/10up/ElasticPress)
 * Find similar terms to merge together using either [OpenAI's Embedding API](https://platform.openai.com/docs/guides/embeddings) or [Microsoft Azure's OpenAI service](https://azure.microsoft.com/en-us/products/ai-services/openai-service) in combination with [ElasticPress](https://github.com/10up/ElasticPress). Note this only compares top-level terms and if you merge a term that has children, these become top-level terms as per default WordPress behavior
 * BETA: Recommend content based on overall site traffic via [Microsoft Azure's AI Personalizer API](https://azure.microsoft.com/en-us/services/cognitive-services/personalizer/) *(note that this service has been [deprecated by Microsoft](https://learn.microsoft.com/en-us/azure/ai-services/personalizer/) and as such, will no longer work. We are looking to replace this with a new provider to maintain the same functionality (see [issue#392](https://github.com/10up/classifai/issues/392))*
-* Generate image alt text, image tags, and smartly crop images using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
-* Scan images and PDF files for embedded text and save for use in post meta using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
+* Generate image alt text using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/), [OpenAI's ChatGPT API](https://platform.openai.com/docs/guides/chat), [xAI's Grok](https://x.ai/) or locally using [Ollama](https://ollama.com/)
+* Generate image tags and extract text from images using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/) or locally using [Ollama](https://ollama.com/)
+* Smartly crop images using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
+* Scan PDF files for embedded text and save for use in post meta using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
 * Bulk classify content with [WP-CLI](https://wp-cli.org/)
 
 ### Language Processing
@@ -55,13 +57,14 @@ Tap into leading cloud-based services like [OpenAI](https://openai.com/), [Micro
 ## Requirements
 
 * PHP 7.4+
-* [WordPress](http://wordpress.org) 6.5+
+* [WordPress](http://wordpress.org) 6.6+
 * To utilize the NLU Language Processing functionality, you will need an active [IBM Watson](https://cloud.ibm.com/registration) account.
 * To utilize the ChatGPT, Embeddings, Text to Speech or Whisper Language Processing functionality or DALL·E Image Processing functionality, you will need an active [OpenAI](https://platform.openai.com/signup) account.
 * To utilize the Azure AI Vision Image Processing functionality or Text to Speech Language Processing functionality, you will need an active [Microsoft Azure](https://signup.azure.com/signup) account.
 * To utilize the Azure OpenAI Language Processing functionality, you will need an active [Microsoft Azure](https://signup.azure.com/signup) account and you will need to [apply](https://aka.ms/oai/access) for OpenAI access.
 * To utilize the Google Gemini Language Processing functionality, you will need an active [Google Gemini](https://ai.google.dev/tutorials/setup) account.
 * To utilize the AWS Language Processing functionality, you will need an active [AWS](https://console.aws.amazon.com/) account.
+* To utilize the Ollama Language or Image Processing functionality, you will need to install [Ollama](https://ollama.com/) and download the appropriate models.
 * To utilize the Smart 404 feature, you will need an active [OpenAI](https://platform.openai.com/signup) account or [Microsoft Azure](https://signup.azure.com/signup) account with OpenAI access and you will need to use [ElasticPress](https://github.com/10up/ElasticPress) 5.0.0+ and [Elasticsearch](https://www.elastic.co/elasticsearch) 7.0+.
 * To utilize the Term Cleanup feature, you will need an active [OpenAI](https://platform.openai.com/signup) account or [Microsoft Azure](https://signup.azure.com/signup) account with OpenAI access. For better performance, you will need [ElasticPress](https://github.com/10up/ElasticPress) 5.0.0+ and [Elasticsearch](https://www.elastic.co/elasticsearch) 7.0+.
 
@@ -709,6 +712,27 @@ For more information, see https://docs.microsoft.com/en-us/azure/cognitive-servi
 
 ### 3. Use "Recommended Content" block to display recommended content on your website
 
+## Run locally hosted LLMs
+
+Some of the Features in ClassifAI can be set up to use locally hosted LLMs. This has the benefit of complete privacy and data control, as well as being able to be run without any cost. The trade-offs here are performance isn't as great and results may also be less accurate.
+
+Right now, this is powered by Ollama, a tool that allows you to host and run LLMs locally. To set this up, follow the steps below:
+
+### 1. Install Ollama
+
+* [Install Ollama](https://ollama.com/) on your local machine.
+* By default Ollama runs at `http://localhost:11434/`.
+
+### 2. Install the model
+
+* Decide which models you want to use. This will depend on the Feature you are setting up. For instance, if you want to use Image Processing Features, ensure you install a Vision model. If you want to use the Classification Feature, ensure you install an Embedding model. All other Features should work with standard models.
+* Install the model locally by running `ollama pull <model-name>` in your terminal.
+
+### 3. Configure Provider
+
+* Once Ollama is running and the model is installed, you can proceed to use it as a Provider for the desired Feature.
+* Note that when using locally hosted LLMs, performance may be slower than using cloud-based services, especially for initial requests. Results may also be less accurate but these are the trade-offs for privacy and data control.
+
 ## WP CLI Commands
 
 - Check out the [ClassifAI docs](https://10up.github.io/classifai/).

classifai.php

@@ -5,7 +5,7 @@
  * Update URI:        https://classifaiplugin.com
  * Description:       Enhance your WordPress content with Artificial Intelligence and Machine Learning services.
  * Version:           3.3.0-dev
- * Requires at least: 6.5
+ * Requires at least: 6.6
  * Requires PHP:      7.4
  * Author:            10up
  * Author URI:        https://10up.com

includes/Classifai/Features/Classification.php

@@ -6,6 +6,7 @@
 use Classifai\Providers\Watson\NLU;
 use Classifai\Providers\OpenAI\Embeddings as OpenAIEmbeddings;
 use Classifai\Providers\Azure\Embeddings as AzureEmbeddings;
+use Classifai\Providers\Localhost\OllamaEmbeddings;
 use WP_REST_Server;
 use WP_REST_Request;
 use WP_Error;
@@ -43,6 +44,7 @@ public function __construct() {
 			NLU::ID              => __( 'IBM Watson NLU', 'classifai' ),
 			OpenAIEmbeddings::ID => __( 'OpenAI Embeddings', 'classifai' ),
 			AzureEmbeddings::ID  => __( 'Azure OpenAI Embeddings', 'classifai' ),
+			OllamaEmbeddings::ID => __( 'Ollama', 'classifai' ),
 		];
 	}
 
@@ -250,6 +252,7 @@ public function save( int $post_id, array $results, bool $link = true ) {
 				break;
 			case AzureEmbeddings::ID:
 			case OpenAIEmbeddings::ID:
+			case OllamaEmbeddings::ID:
 				$results = $provider_instance->set_terms( $post_id, $results, $link );
 				break;
 		}
@@ -782,7 +785,7 @@ public function add_custom_settings_fields() {
 		);
 
 		// Embeddings only supports existing terms.
-		if ( isset( $settings['provider'] ) && ( OpenAIEmbeddings::ID === $settings['provider'] || AzureEmbeddings::ID === $settings['provider'] ) ) {
+		if ( isset( $settings['provider'] ) && ( OpenAIEmbeddings::ID === $settings['provider'] || AzureEmbeddings::ID === $settings['provider'] || OllamaEmbeddings::ID === $settings['provider'] ) ) {
 			unset( $method_options['recommended_terms'] );
 			$settings['classification_method'] = 'existing_terms';
 		}
@@ -879,7 +882,7 @@ public function sanitize_default_feature_settings( array $new_settings ): array
 		$new_settings['classification_method'] = sanitize_text_field( $new_settings['classification_method'] ?? $settings['classification_method'] );
 
 		// Embeddings only supports existing terms.
-		if ( isset( $new_settings['provider'] ) && ( OpenAIEmbeddings::ID === $new_settings['provider'] || AzureEmbeddings::ID === $new_settings['provider'] ) ) {
+		if ( isset( $new_settings['provider'] ) && ( OpenAIEmbeddings::ID === $new_settings['provider'] || AzureEmbeddings::ID === $new_settings['provider'] || OllamaEmbeddings::ID === $settings['provider'] ) ) {
 			$new_settings['classification_method'] = 'existing_terms';
 		}
 

includes/Classifai/Features/ContentResizing.php

@@ -7,6 +7,7 @@
 use Classifai\Providers\OpenAI\ChatGPT;
 use Classifai\Providers\Browser\ChromeAI;
 use Classifai\Providers\XAI\Grok;
+use Classifai\Providers\Localhost\Ollama;
 use Classifai\Services\LanguageProcessing;
 use WP_REST_Server;
 use WP_REST_Request;
@@ -56,6 +57,7 @@ public function __construct() {
 			OpenAI::ID    => __( 'Azure OpenAI', 'classifai' ),
 			Grok::ID      => __( 'xAI Grok', 'classifai' ),
 			ChromeAI::ID  => __( 'Chrome AI (experimental)', 'classifai' ),
+			Ollama::ID    => __( 'Ollama', 'classifai' ),
 		];
 	}
 

includes/Classifai/Features/DescriptiveTextGenerator.php

@@ -5,6 +5,7 @@
 use Classifai\Providers\Azure\ComputerVision;
 use Classifai\Providers\OpenAI\ChatGPT;
 use Classifai\Providers\XAI\Grok;
+use Classifai\Providers\Localhost\OllamaMultimodal as OllamaMM;
 use Classifai\Services\ImageProcessing;
 use WP_REST_Server;
 use WP_REST_Request;
@@ -44,6 +45,7 @@ public function __construct() {
 			ComputerVision::ID => __( 'Microsoft Azure AI Vision', 'classifai' ),
 			ChatGPT::ID        => __( 'OpenAI', 'classifai' ),
 			Grok::ID           => __( 'xAI Grok', 'classifai' ),
+			OllamaMM::ID       => __( 'Ollama', 'classifai' ),
 		];
 	}
 
@@ -420,6 +422,15 @@ public function get_settings( $index = false ) {
 			}
 		}
 
+		if ( $settings && ! empty( $settings[ OllamaMM::ID ]['prompt'] ) ) {
+			foreach ( $settings[ OllamaMM::ID ]['prompt'] as $key => $prompt ) {
+				if ( 1 === intval( $prompt['original'] ) ) {
+					$settings[ OllamaMM::ID ]['prompt'][ $key ]['prompt'] = $this->prompt;
+					break;
+				}
+			}
+		}
+
 		return $settings;
 	}
 

includes/Classifai/Features/ExcerptGeneration.php

@@ -8,6 +8,7 @@
 use Classifai\Providers\OpenAI\ChatGPT;
 use Classifai\Providers\Azure\OpenAI;
 use Classifai\Providers\Browser\ChromeAI;
+use Classifai\Providers\Localhost\Ollama;
 use WP_REST_Server;
 use WP_REST_Request;
 use WP_Error;
@@ -49,6 +50,7 @@ public function __construct() {
 			OpenAI::ID    => __( 'Azure OpenAI', 'classifai' ),
 			Grok::ID      => __( 'xAI Grok', 'classifai' ),
 			ChromeAI::ID  => __( 'Chrome AI (experimental)', 'classifai' ),
+			Ollama::ID    => __( 'Ollama', 'classifai' ),
 		];
 	}
 

includes/Classifai/Features/ImageTagsGenerator.php

@@ -3,6 +3,7 @@
 namespace Classifai\Features;
 
 use Classifai\Providers\Azure\ComputerVision;
+use Classifai\Providers\Localhost\OllamaMultimodal as OllamaMM;
 use Classifai\Services\ImageProcessing;
 use WP_REST_Server;
 use WP_REST_Request;
@@ -22,6 +23,20 @@ class ImageTagsGenerator extends Feature {
 	 */
 	const ID = 'feature_image_tags_generator';
 
+	// phpcs:disable Squiz.PHP.Heredoc.NotAllowed
+	/**
+	 * Prompt for generating tags.
+	 *
+	 * @var string
+	 */
+	public $prompt = <<<EOD
+You are an assistant that generates image tags. You will be provided with an image and will generate a list of tags that best represent the image. Ensure the tags are short. Return at most the best 5 tags and return these in the following format:
+- Tag
+- Another tag
+- ...
+EOD;
+	// phpcs:enable Squiz.PHP.Heredoc.NotAllowed
+
 	/**
 	 * Constructor.
 	 */
@@ -34,6 +49,7 @@ public function __construct() {
 		// Contains just the providers this feature supports.
 		$this->supported_providers = [
 			ComputerVision::ID => __( 'Microsoft Azure AI Vision', 'classifai' ),
+			OllamaMM::ID       => __( 'Ollama', 'classifai' ),
 		];
 	}
 
@@ -47,6 +63,28 @@ public function setup() {
 		add_action( 'rest_api_init', [ $this, 'register_endpoints' ] );
 	}
 
+	/**
+	 * Returns the settings for the feature.
+	 *
+	 * @param string $index The index of the setting to return.
+	 * @return array|mixed
+	 */
+	public function get_settings( $index = false ) {
+		$settings = parent::get_settings( $index );
+
+		// Keep using the original prompt from the codebase to allow updates.
+		if ( $settings && ! empty( $settings[ OllamaMM::ID ]['prompt'] ) ) {
+			foreach ( $settings[ OllamaMM::ID ]['prompt'] as $key => $prompt ) {
+				if ( 1 === intval( $prompt['original'] ) ) {
+					$settings[ OllamaMM::ID ]['prompt'][ $key ]['prompt'] = $this->prompt;
+					break;
+				}
+			}
+		}
+
+		return $settings;
+	}
+
 	/**
 	 * Set up necessary hooks.
 	 */

includes/Classifai/Features/ImageTextExtraction.php

@@ -3,6 +3,7 @@
 namespace Classifai\Features;
 
 use Classifai\Providers\Azure\ComputerVision;
+use Classifai\Providers\Localhost\OllamaMultimodal as OllamaMM;
 use Classifai\Services\ImageProcessing;
 use WP_REST_Server;
 use WP_REST_Request;
@@ -23,6 +24,13 @@ class ImageTextExtraction extends Feature {
 	 */
 	const ID = 'feature_image_to_text_generator';
 
+	/**
+	 * Prompt for extracting text.
+	 *
+	 * @var string
+	 */
+	public $prompt = 'You are an assistant that extracts text from images. You will be provided with an image and will return whatever text is in the image. Return only the text, nothing else. If there is no text present, return the word none.';
+
 	/**
 	 * Constructor.
 	 */
@@ -35,6 +43,7 @@ public function __construct() {
 		// Contains just the providers this feature supports.
 		$this->supported_providers = [
 			ComputerVision::ID => __( 'Microsoft Azure AI Vision', 'classifai' ),
+			OllamaMM::ID       => __( 'Ollama', 'classifai' ),
 		];
 	}
 
@@ -48,6 +57,28 @@ public function setup() {
 		add_action( 'rest_api_init', [ $this, 'register_endpoints' ] );
 	}
 
+	/**
+	 * Returns the settings for the feature.
+	 *
+	 * @param string $index The index of the setting to return.
+	 * @return array|mixed
+	 */
+	public function get_settings( $index = false ) {
+		$settings = parent::get_settings( $index );
+
+		// Keep using the original prompt from the codebase to allow updates.
+		if ( $settings && ! empty( $settings[ OllamaMM::ID ]['prompt'] ) ) {
+			foreach ( $settings[ OllamaMM::ID ]['prompt'] as $key => $prompt ) {
+				if ( 1 === intval( $prompt['original'] ) ) {
+					$settings[ OllamaMM::ID ]['prompt'][ $key ]['prompt'] = $this->prompt;
+					break;
+				}
+			}
+		}
+
+		return $settings;
+	}
+
 	/**
 	 * Set up necessary hooks.
 	 */

includes/Classifai/Features/KeyTakeaways.php

@@ -3,11 +3,9 @@
 namespace Classifai\Features;
 
 use Classifai\Services\LanguageProcessing;
-use Classifai\Providers\XAI\Grok;
-use Classifai\Providers\GoogleAI\GeminiAPI;
 use Classifai\Providers\OpenAI\ChatGPT;
 use Classifai\Providers\Azure\OpenAI;
-use Classifai\Providers\Browser\ChromeAI;
+use Classifai\Providers\Localhost\Ollama;
 use WP_REST_Server;
 use WP_REST_Request;
 use WP_Error;
@@ -47,6 +45,7 @@ public function __construct() {
 		$this->supported_providers = [
 			ChatGPT::ID => __( 'OpenAI ChatGPT', 'classifai' ),
 			OpenAI::ID  => __( 'Azure OpenAI', 'classifai' ),
+			Ollama::ID  => __( 'Ollama', 'classifai' ),
 		];
 	}