setup.sh

#!/bin/bash

# Setup script for AI-Assisted Web Crawling Platform
# This script automates the installation and setup process

set -e  # Exit on error

echo "🤖 AI-Assisted Web Crawling Platform - Setup"
echo "============================================="
echo ""

# Check Python version
echo "📋 Checking Python version..."
python_version=$(python3 --version 2>&1 | awk '{print $2}')
echo "   Found Python $python_version"

# Check if Python 3.9+
required_version="3.9"
if [ "$(printf '%s\n' "$required_version" "$python_version" | sort -V | head -n1)" != "$required_version" ]; then
    echo "❌ Error: Python 3.9 or higher is required"
    exit 1
fi
echo "   ✅ Python version OK"
echo ""

# Create virtual environment
echo "🔧 Creating virtual environment..."
if [ ! -d "venv" ]; then
    python3 -m venv venv
    echo "   ✅ Virtual environment created"
else
    echo "   ℹ️  Virtual environment already exists"
fi
echo ""

# Activate virtual environment
echo "🔌 Activating virtual environment..."
source venv/bin/activate
echo "   ✅ Virtual environment activated"
echo ""

# Upgrade pip, setuptools, and wheel (critical for pandas)
echo "⬆️  Upgrading pip, setuptools, and wheel..."
pip install --upgrade pip setuptools wheel
echo "   ✅ Build tools upgraded"
echo ""

# Install Python dependencies
echo "📦 Installing Python dependencies..."
echo "   (This may take a few minutes, especially for pandas...)"
pip install -r requirements.txt
echo "   ✅ Python dependencies installed"
echo ""

# Install Playwright browsers
echo "🌐 Installing Playwright browsers..."
playwright install chromium
echo "   ✅ Playwright browsers installed"
echo ""

# Create .env file if it doesn't exist
echo "⚙️  Setting up configuration..."
if [ ! -f ".env" ]; then
    cp .env.example .env
    echo "   ✅ Created .env file from template"
    echo ""
    echo "   ⚠️  IMPORTANT: Edit .env and add your API key!"
    echo "   Example: OPENAI_API_KEY=sk-your-key-here"
else
    echo "   ℹ️  .env file already exists"
fi
echo ""

# Create output directories
echo "📁 Creating output directories..."
mkdir -p outputs reports
echo "   ✅ Output directories created"
echo ""

echo "============================================="
echo "✅ Setup Complete!"
echo ""
echo "📝 Next Steps:"
echo ""
echo "1. Edit .env file and add your API key:"
echo "   nano .env"
echo ""
echo "2. Make sure you have an API key for one of:"
echo "   - OpenAI (OPENAI_API_KEY)"
echo "   - Azure OpenAI (AZURE_API_KEY, AZURE_API_BASE)"
echo "   - Anthropic (ANTHROPIC_API_KEY)"
echo ""
echo "3. Run the application:"
echo "   source venv/bin/activate  # If not already activated"
echo "   streamlit run app.py"
echo ""
echo "4. Open browser to: http://localhost:8501"
echo ""
echo "🎉 Happy crawling!"