cbmc/codedetect/.github/workflows/comprehensive-tests.yml

name: Comprehensive Tests

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]
  schedule:
    - cron: '0 2 * * *'  # 每天凌晨2点运行

jobs:
  unit-tests:
    name: Unit Tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        python-version: [3.9, 3.10, 3.11]
        fail-fast: false

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache pip packages
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-          

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt        

    - name: Run unit tests
      run: |
        python -m pytest tests/unit/ -v --tb=short --junitxml=test-results/unit.xml        
      env:
        PYTHONPATH: ${{ github.workspace }}

    - name: Upload test results
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: unit-test-results-${{ matrix.os }}-${{ matrix.python-version }}
        path: test-results/unit.xml

  integration-tests:
    name: Integration Tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        python-version: [3.9, 3.10]
        fail-fast: false

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache pip packages
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-          

    - name: Install system dependencies
      run: |
        if [ "$RUNNER_OS" == "Linux" ]; then
          sudo apt-get update
          sudo apt-get install -y cbmc build-essential
        elif [ "$RUNNER_OS" == "macOS" ]; then
          brew install cbmc
        fi        

    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt        

    - name: Setup FreeRTOS
      run: |
        ./scripts/setup-freertos-example.sh        
      continue-on-error: true

    - name: Run integration tests
      run: |
        python -m pytest tests/integration/ -v --tb=short --junitxml=test-results/integration.xml        
      env:
        PYTHONPATH: ${{ github.workspace }}
        FREERTOS_PATH: /opt/freertos

    - name: Upload test results
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: integration-test-results-${{ matrix.os }}-${{ matrix.python-version }}
        path: test-results/integration.xml

  regression-tests:
    name: Regression Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.9, 3.10]
        fail-fast: false

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache pip packages
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-          

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt        

    - name: Run regression tests
      run: |
        python -m pytest tests/regression/ -v --tb=short --junitxml=test-results/regression.xml        
      env:
        PYTHONPATH: ${{ github.workspace }}

    - name: Upload test results
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: regression-test-results-${{ matrix.python-version }}
        path: test-results/regression.xml

  performance-tests:
    name: Performance Tests
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || contains(github.event.head_commit.message, '[performance]')

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: 3.10

    - name: Cache pip packages
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-          

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt        

    - name: Run performance tests
      run: |
        python -m pytest tests/performance/ -v --tb=short --junitxml=test-results/performance.xml -m "not slow"        
      env:
        PYTHONPATH: ${{ github.workspace }}
      timeout-minutes: 30

    - name: Upload test results
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: performance-test-results
        path: test-results/performance.xml

  code-quality:
    name: Code Quality
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: 3.10

    - name: Cache pip packages
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-          

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt        

    - name: Run flake8
      run: |
        flake8 src/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
        flake8 src/ tests/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics        

    - name: Run black check
      run: |
        black --check --diff src/ tests/        

    - name: Run isort check
      run: |
        isort --check-only --diff src/ tests/        

    - name: Run mypy
      run: |
        mypy src/ --ignore-missing-imports        

    - name: Run bandit security check
      run: |
        bandit -r src/ -f json -o bandit-report.json        
      continue-on-error: true

    - name: Upload security report
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: security-report
        path: bandit-report.json

  coverage:
    name: Coverage Report
    runs-on: ubuntu-latest
    needs: [unit-tests, integration-tests, regression-tests]

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: 3.10

    - name: Cache pip packages
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-          

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt        

    - name: Download all test results
      uses: actions/download-artifact@v3
      with:
        path: test-results/

    - name: Run tests with coverage
      run: |
        python -m pytest tests/ --cov=src/ --cov-report=xml --cov-report=html --cov-report=term-missing        
      env:
        PYTHONPATH: ${{ github.workspace }}

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml
        flags: unittests
        name: codecov-umbrella
        fail_ci_if_error: false

    - name: Upload coverage report
      uses: actions/upload-artifact@v3
      with:
        name: coverage-report
        path: htmlcov/

  benchmark:
    name: Benchmark
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || contains(github.event.head_commit.message, '[benchmark]')

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: 3.10

    - name: Cache pip packages
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-          

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt        

    - name: Run benchmarks
      run: |
        python tests/tools/benchmark_runner.py        
      env:
        PYTHONPATH: ${{ github.workspace }}

    - name: Upload benchmark results
      uses: actions/upload-artifact@v3
      with:
        name: benchmark-results
        path: benchmark_results/

  test-summary:
    name: Test Summary
    runs-on: ubuntu-latest
    needs: [unit-tests, integration-tests, regression-tests, performance-tests, code-quality, coverage]
    if: always()

    steps:
    - uses: actions/checkout@v4

    - name: Download all test results
      uses: actions/download-artifact@v3
      with:
        path: all-test-results/

    - name: Generate test summary
      run: |
        ./scripts/run-comprehensive-tests.sh -t unit,integration,regression        
      continue-on-error: true

    - name: Upload test summary
      uses: actions/upload-artifact@v3
      with:
        name: test-summary
        path: test_results/