handling multimodal entries

Author: ajosh0504

apps/interactive-journal/backend/app/routers/helpers.py

@@ -1,11 +1,14 @@
+import base64
 import logging
 import uuid
 from datetime import datetime, timedelta
+from io import BytesIO
 from pathlib import Path
 
 from fastapi import UploadFile
+from PIL import Image
 
-from app.config import USER_ID, VECTOR_INDEX_NAME, VECTOR_NUM_CANDIDATES
+from app.config import IMAGE_SIZE, USER_ID, VECTOR_INDEX_NAME, VECTOR_NUM_CANDIDATES
 from app.services.anthropic import extract_memories
 from app.services.voyage import get_multimodal_embedding, get_text_embedding
 
@@ -18,20 +21,6 @@
 UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
 
 
-def get_conversation_history(db, entry_id: str) -> list[dict]:
-    """Get conversation history for an entry."""
-    history = list(
-        db.messages.find(
-            {"entry_id": entry_id}, {"role": 1, "content": 1, "_id": 0}
-        ).sort("created_at", 1)
-    )
-    return [
-        {"role": msg["role"], "content": msg["content"]}
-        for msg in history
-        if msg.get("content")
-    ]
-
-
 def save_user_message(
     db, entry_id: str, content: str | Path, version: int, msg_date: datetime
 ) -> None:
@@ -62,6 +51,28 @@ def save_user_message(
     logger.info(f"Saved message for entry {entry_id}")
 
 
+def extract_and_save_memories(
+    db, entry_id: str, conversation: list[dict], entry_date: datetime
+) -> None:
+    """Extract memories from conversation and save them."""
+    context = "\n".join(f"{msg['role']}: {msg['content']}" for msg in conversation)
+    memories = extract_memories(context)
+
+    if memories:
+        memory_docs = [
+            {
+                "user_id": USER_ID,
+                "entry_id": entry_id,
+                "content": memory_content,
+                "embedding": get_text_embedding(memory_content, input_type="document"),
+                "created_at": entry_date,
+            }
+            for memory_content in memories
+        ]
+        db.memories.insert_many(memory_docs)
+        logger.info(f"Extracted and saved {len(memories)} memories: {memories}")
+
+
 def retrieve_relevant_memories(db, query: str) -> list[str]:
     """Retrieve relevant memories via vector search."""
     query_embedding = get_text_embedding(query, input_type="query")
@@ -84,26 +95,42 @@ def retrieve_relevant_memories(db, query: str) -> list[str]:
     return memories
 
 
-def extract_and_save_memories(
-    db, entry_id: str, conversation: list[dict], entry_date: datetime
-) -> None:
-    """Extract memories from conversation and save them."""
-    context = "\n".join(f"{msg['role']}: {msg['content']}" for msg in conversation)
-    memories = extract_memories(context)
+def get_conversation_history(db, entry_id: str, include_images: bool = True) -> list[dict]:
+    """Get conversation history for an entry."""
+    history = list(
+        db.messages.find(
+            {"entry_id": entry_id}, {"role": 1, "content": 1, "image": 1, "_id": 0}
+        ).sort("created_at", 1)
+    )
 
-    if memories:
-        memory_docs = [
-            {
-                "user_id": USER_ID,
-                "entry_id": entry_id,
-                "content": memory_content,
-                "embedding": get_text_embedding(memory_content, input_type="document"),
-                "created_at": entry_date,
-            }
-            for memory_content in memories
-        ]
-        db.memories.insert_many(memory_docs)
-        logger.info(f"Extracted and saved {len(memories)} memories: {memories}")
+    messages = []
+    for msg in history:
+        if msg.get("content"):
+            messages.append({"role": msg["role"], "content": msg["content"]})
+        elif msg.get("image") and include_images:
+            image_path = UPLOADS_DIR / msg["image"]
+            if image_path.exists():
+                messages.append(
+                    {
+                        "role": msg["role"],
+                        "content": [image_to_base64(image_path)],
+                    }
+                )
+    return messages
+
+
+def image_to_base64(image_path: Path) -> dict:
+    """Convert an image file to Claude's base64 format, resizing to fit limits."""
+    with Image.open(image_path) as img:
+        img = img.resize(IMAGE_SIZE, Image.Resampling.LANCZOS)
+        buffer = BytesIO()
+        img.save(buffer, format="JPEG", quality=85)
+        data = base64.standard_b64encode(buffer.getvalue()).decode("utf-8")
+
+    return {
+        "type": "image",
+        "source": {"type": "base64", "media_type": "image/jpeg", "data": data},
+    }
 
 
 def save_assistant_message(db, entry_id: str, content: str, msg_date: datetime) -> None:
@@ -120,7 +147,7 @@ def save_assistant_message(db, entry_id: str, content: str, msg_date: datetime)
 
 
 def save_image_file(image_file: UploadFile) -> Path:
-    """Save uploaded image file and return the path."""
+    """Save uploaded image file."""
     filename = f"{uuid.uuid4()}{Path(image_file.filename).suffix or '.jpg'}"
     image_path = UPLOADS_DIR / filename
     with open(image_path, "wb") as f:

apps/interactive-journal/backend/app/routers/routes.py

@@ -13,6 +13,7 @@
     get_mood_distribution,
     get_themes,
     get_total_entries,
+    image_to_base64,
     retrieve_relevant_memories,
     save_assistant_message,
     save_image_file,
@@ -62,10 +63,17 @@ def send_message(
     # Save image files to disk before streaming (file handles close after)
     image_paths = [save_image_file(image) for image in images]
 
+    # Build current message (text, images, or both)
+    messages = []
+    if content:
+        messages.append({"type": "text", "text": content})
+    for path in image_paths:
+        messages.append(image_to_base64(path))
+
     # Get conversation history and add current message
     conversation = get_conversation_history(db, entry_id)
-    if content:
-        conversation.append({"role": "user", "content": content})
+    if messages:
+        conversation.append({"role": "user", "content": messages})
 
     # Retrieve relevant memories for context (V2 only)
     memories = retrieve_relevant_memories(db, content) if is_v2 and content else []
@@ -144,7 +152,7 @@ def search_entries(q: str, version: int = 1):
 def save_entry(entry_id: str, entry_date: str = Form(...)):
     """Analyze entry for sentiment/themes and extract memories."""
     db = get_database()
-    conversation = get_conversation_history(db, entry_id)
+    conversation = get_conversation_history(db, entry_id, include_images=False)
 
     if not conversation:
         return {"error": "No messages in entry"}

apps/interactive-journal/frontend/src/components/Entry.jsx

@@ -32,6 +32,7 @@ function Entry({ messages, onSendMessage, hasActiveEntry, activeEntry, entries,
   }
 
   const handleSaveEntry = async () => {
+    setSaveStatus('saving')
     try {
       const activeEntryObj = entries.find(e => e._id === activeEntry)
       const formData = new FormData()
@@ -45,6 +46,7 @@ function Entry({ messages, onSendMessage, hasActiveEntry, activeEntry, entries,
       setTimeout(() => setSaveStatus(null), 2000)
     } catch (error) {
       console.error('Failed to save entry:', error)
+      setSaveStatus(null)
     }
   }
 
@@ -291,10 +293,11 @@ function Entry({ messages, onSendMessage, hasActiveEntry, activeEntry, entries,
         {isV2 && messages.length > 0 && (
           <div className="save-entry">
             <button
-              className={`save-btn ${saveStatus === 'saved' ? 'saved' : ''}`}
+              className={`save-btn ${saveStatus || ''}`}
               onClick={handleSaveEntry}
+              disabled={saveStatus === 'saving'}
             >
-              {saveStatus === 'saved' ? 'Saved ✓' : 'Save'}
+              {saveStatus === 'saving' ? 'Saving...' : saveStatus === 'saved' ? 'Saved ✓' : 'Save'}
             </button>
           </div>
         )}