ruby.yml

name: Ruby Tests
on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - '*'
jobs:
  test:
    runs-on: ubuntu-latest
    # TODO: explore ways to test the matrix across several Ruby versions instead of a single .ruby-version pin.
    strategy:
      fail-fast: false
      matrix:
        include:
        - name: DSPy Core
          gem: dspy
          command: bundle exec rspec --format documentation --pattern 'spec/unit/**/*_spec.rb'
            --pattern 'spec/integration/ruby_llm_integration_spec.rb'
            --pattern 'spec/integration/dspy/lm/adapters/ruby_llm_adapter_spec.rb'
            --pattern 'spec/integration/dspy/lm/adapters/gemini_adapter_spec.rb'
            --pattern 'spec/integration/dspy/predict_validation_spec.rb'
            --pattern 'spec/integration/dspy/tools/github_cli_toolset_spec.rb'
            --pattern 'spec/integration/ollama_integration_spec.rb'
            --exclude-pattern '**/spec/unit/dspy/datasets/**/*_spec.rb' --exclude-pattern
            '**/spec/unit/dspy/teleprompt/mipro_v2_*' --exclude-pattern '**/spec/unit/teleprompt/mipro_v2_*'
            --exclude-pattern '**/spec/unit/optimizers/gaussian_process_spec.rb' --exclude-pattern
            '**/spec/integration/dspy/datasets/**/*_spec.rb' --exclude-pattern '**/spec/integration/dspy/mipro_*'
            --exclude-pattern '**/spec/unit/dspy/code_act/**/*_spec.rb' --exclude-pattern '**/spec/integration/dspy/code_act*_spec.rb'
            --exclude-pattern '**/spec/unit/dspy/observability/**/*_spec.rb' --exclude-pattern '**/spec/integration/dspy/observability*_spec.rb'
            --exclude-pattern '**/spec/integration/langfuse_*'
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy DeepSearch
          gem: dspy-deep_search
          command: bundle exec rspec --format documentation spec/unit/deep_search spec/integration/deep_search
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy DeepResearch
          gem: dspy-deepresearch
          command: bundle exec rspec --format documentation spec/unit/deep_research spec/integration/deep_research
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy CodeAct
          gem: dspy-code_act
          command: bundle exec rspec --format documentation spec/unit/dspy/code_act spec/integration/dspy/code_act_spec.rb
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '1'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy Datasets
          gem: dspy-datasets
          command: bundle exec rspec --format documentation --pattern 'spec/unit/dspy/datasets/**/*_spec.rb'
            --pattern 'spec/integration/dspy/datasets/**/*_spec.rb'
          install_arrow: 'true'
          datasets: '1'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy GEPA
          gem: dspy-gepa
          command: bundle exec rspec --format documentation spec/unit/gepa spec/unit/teleprompt/gepa_spec.rb
            spec/integration/dspy/teleprompt/gepa_smoke_spec.rb
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '1'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy MIPROv2
          gem: dspy-miprov2
          command: bundle exec rspec --format documentation --pattern 'spec/unit/optimizers/gaussian_process_spec.rb'
            --pattern 'spec/unit/dspy/teleprompt/mipro_v2_*' --pattern 'spec/unit/teleprompt/mipro_v2_*'
            --pattern 'spec/integration/dspy/mipro_v2*_spec.rb'
          install_openblas: 'true'
          datasets: '0'
          miprov2: '1'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy Evaluations
          gem: dspy-evals
          command: bundle exec rspec --format documentation spec/integration/dspy/evaluate_spec.rb spec/integration/evaluation_integration_spec.rb
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy Schema
          gem: dspy-schema
          command: bundle exec ruby -Ilib -e "require 'sorbet-runtime'; require 'dspy/schema'; schema = DSPy::TypeSystem::SorbetJsonSchema.type_to_json_schema(T.nilable(String)); unless schema[:type].is_a?(Array) && schema[:type].include?('null'); abort('nilable schema missing null type'); end"
          datasets: '0'
          miprov2: '0'
          evals: '0'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: Sorbet::Toon Specs
          gem: sorbet-toon
          command: bundle exec rspec --format documentation spec/sorbet/toon
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
        - name: DSPy O11y Langfuse
          gem: dspy-o11y-langfuse
          command: bundle exec rspec --format documentation spec/unit/dspy/observability/observation_type_spec.rb spec/integration/dspy/observability_spec.rb spec/integration/dspy/span_nesting_spec.rb spec/integration/langfuse_multi_span_spec.rb
          datasets: '0'
          miprov2: '0'
          evals: '1'
          code_act: '0'
          schema: '1'
          gepa: '0'
          o11y: '1'
          o11y_langfuse: '1'
    name: Run tests (${{ matrix.name }})
    steps:
    - uses: actions/checkout@v4
    - name: Install Arrow dependencies
      if: ${{ matrix.install_arrow == 'true' }}
      run: |
        sudo apt-get update
        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
          ca-certificates lsb-release wget build-essential pkg-config \
          gobject-introspection libgirepository1.0-dev libffi-dev \
          libglib2.0-dev zlib1g-dev liblzma-dev
        wget "https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb"
        sudo apt-get install -y "./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb"
        sudo apt-get update
        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
          libarrow-glib-dev libparquet-glib-dev
    - name: Install OpenBLAS/LAPACK
      if: ${{ matrix.install_openblas == 'true' }}
      run: |
        sudo apt-get update
        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
          libopenblas-dev liblapacke-dev gfortran
    - name: Read Ruby version from .ruby-version
      id: ruby-version
      run: echo "version=$(cat .ruby-version)" >> $GITHUB_OUTPUT
    - name: Cache gems
      uses: actions/cache@v4
      with:
        path: vendor/bundle
        key: gems-${{ runner.os }}-${{ matrix.datasets }}-${{ matrix.miprov2 }}-${{ matrix.evals }}-${{ matrix.code_act }}-${{ matrix.schema }}-${{ matrix.gepa }}-${{ matrix.o11y }}-${{ matrix.o11y_langfuse }}-${{ hashFiles('**/Gemfile.lock') }}
        restore-keys: |
          gems-${{ runner.os }}-${{ matrix.datasets }}-${{ matrix.miprov2 }}-${{ matrix.evals }}-${{ matrix.code_act }}-${{ matrix.schema }}-${{ matrix.gepa }}-${{ matrix.o11y }}-${{ matrix.o11y_langfuse }}-
    - name: Set up Ruby
      uses: ruby/setup-ruby@v1
      with:
        ruby-version: ${{ steps.ruby-version.outputs.version }}
        bundler-cache: false
    - name: Install dependencies
      env:
        DSPY_WITH_DATASETS: ${{ matrix.datasets }}
        DSPY_WITH_MIPROV2: ${{ matrix.miprov2 }}
        DSPY_WITH_EVALS: ${{ matrix.evals }}
        DSPY_WITH_CODE_ACT: ${{ matrix.code_act }}
        DSPY_WITH_SCHEMA: ${{ matrix.schema }}
        DSPY_WITH_GEPA: ${{ matrix.gepa }}
        DSPY_WITH_O11Y: ${{ matrix.o11y }}
        DSPY_WITH_O11Y_LANGFUSE: ${{ matrix.o11y_langfuse }}
      run: |
        bundle config set --local path vendor/bundle
        bundle install --jobs 4
    - name: Run tests
      run: ${{ matrix.command }}
      env:
        DSPY_WITH_DATASETS: ${{ matrix.datasets }}
        DSPY_WITH_MIPROV2: ${{ matrix.miprov2 }}
        DSPY_WITH_EVALS: ${{ matrix.evals }}
        DSPY_WITH_CODE_ACT: ${{ matrix.code_act }}
        DSPY_WITH_SCHEMA: ${{ matrix.schema }}
        DSPY_WITH_GEPA: ${{ matrix.gepa }}
        DSPY_WITH_O11Y: ${{ matrix.o11y }}
        DSPY_WITH_O11Y_LANGFUSE: ${{ matrix.o11y_langfuse }}
        OPENAI_API_KEY: test-openai-key-for-vcr
        ANTHROPIC_API_KEY: test-anthropic-key-for-vcr
        GEMINI_API_KEY: test-gemini-key-for-vcr
        OPENROUTER_API_KEY: test-openrouter-key-for-vcr
        EXA_API_KEY: test-exa-key-for-vcr
        LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
        LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}