GitHub Actions

name: Eval Check

on: [pull_request]

jobs:
  eval-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '18'
      
      - name: Run evals
        run: npm run evals  # Your eval command
      
      - name: Install Geval
        run: npm install -g @geval-labs/cli
      
      - name: Check eval results
        run: |
          geval check \
            --contract contract.yaml \
            --eval eval-results.csv

name: Eval Check

on:
  pull_request:
    paths:
      - 'prompts/**'
      - 'agents/**'

jobs:
  eval-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Install Geval
        run: npm install -g @geval-labs/cli

      - name: Run Evals
        run: |
          # Your eval command here (promptfoo, langsmith, etc.)
          npm run evals -- --output eval-results.json

      - name: Validate Contract
        run: geval validate contract.yaml

      - name: Check Eval Results
        run: |
          geval check \
            --contract contract.yaml \
            --eval eval-results.json \
            --json > decision.json

      - name: Comment on PR
        if: always()
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const decision = JSON.parse(fs.readFileSync('decision.json', 'utf8'));
            
            const icon = decision.status === 'PASS' ? '✅' : '❌';
            const body = `## ${icon} Geval: ${decision.status}\n\n${decision.summary}`;
            
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: body
            });

- name: Check Multiple Evals
  run: |
    geval check \
      --contract contract.yaml \
      --eval results1.json \
      --eval results2.json \
      --eval results3.csv

- name: Download Baseline
  uses: actions/download-artifact@v3
  with:
    name: baseline-results
    path: ./baseline

- name: Check with Baseline
  run: |
    geval check \
      --contract contract.yaml \
      --eval current-results.json \
      --baseline baseline/results.json

- name: Cache node modules
  uses: actions/cache@v3
  with:
    path: ~/.npm
    key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
    restore-keys: |
      ${{ runner.os }}-node-

strategy:
  matrix:
    node-version: [18, 20, 22]
steps:
  - uses: actions/setup-node@v4
    with:
      node-version: ${{ matrix.node-version }}