LLM Security Scanner
Run scanner ↗
Scanning

CI/CD Integration

Run the scanner as a gate in GitHub Actions or GitLab CI. Block merges when the risk score exceeds your threshold.

GitHub Actions

# .github/workflows/llm-security.yml
name: LLM Security Scan
on: [pull_request, workflow_dispatch]

jobs:
  scan:
    runs-on: ubuntu-latest
    env:
      LLM_ENDPOINT: ${{ vars.LLM_ENDPOINT }}      # URL reachable from this job
      LLM_JUDGE_MODEL: ${{ vars.LLM_JUDGE_MODEL || 'llama3.2:3b' }}
      LLM_FAIL_ON_SCORE: ${{ vars.LLM_FAIL_ON_SCORE || '7.0' }}
      LLM_SEVERITY: ${{ vars.LLM_SEVERITY }}
      LLM_CATEGORIES: ${{ vars.LLM_CATEGORIES }}
      LLM_INCLUDE_DOS_TESTS: ${{ vars.LLM_INCLUDE_DOS_TESTS || 'false' }}
    steps:
      - uses: actions/checkout@v4
      - name: Validate configuration
        run: test -n "$LLM_ENDPOINT"
      - name: Start Ollama
        run: |
          curl -fsSL https://ollama.com/install.sh | sh
          ollama serve &
          sleep 5
          ollama pull "$LLM_JUDGE_MODEL"
      - uses: konradxmalinowski/llm-security-scanner/.github/actions/llm-scan@main
        with:
          target: ${{ env.LLM_ENDPOINT }}
          target-type: url
          judge-model: ${{ env.LLM_JUDGE_MODEL }}
          severity: ${{ env.LLM_SEVERITY }}
          categories: ${{ env.LLM_CATEGORIES }}
          include-dos-tests: ${{ env.LLM_INCLUDE_DOS_TESTS }}
          fail-on-score: ${{ env.LLM_FAIL_ON_SCORE }}
          output-dir: ./reports
      # Add api-key: ${{ secrets.LLM_API_KEY }} only if the target requires bearer auth
      - uses: actions/upload-artifact@v4
        with:
          name: llm-security-reports
          path: reports/

GitLab CI

# .gitlab-ci.yml
llm-security-scan:
  image: python:3.11-slim
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "web"
  services:
    - name: ollama/ollama:latest
      alias: ollama
  variables:
    OLLAMA_HOST: http://ollama:11434
    LLM_ENDPOINT: ""  # no default on purpose - set as a CI/CD variable
    LLM_JUDGE_MODEL: llama3.2:3b
    LLM_FAIL_ON_SCORE: "7.0"
    LLM_SEVERITY: ""
    LLM_CATEGORIES: ""
    LLM_INCLUDE_DOS_TESTS: "false"
  before_script:
    - test -n "$LLM_ENDPOINT"
    - pip install uv
    - uv pip install --system "git+https://github.com/konradxmalinowski/llm-security-scanner.git@main"
    - curl -fsS "$OLLAMA_HOST/api/version"
    - curl -fsS "$OLLAMA_HOST/api/pull" -d "{\"name\":\"$LLM_JUDGE_MODEL\"}"
  script:
    - |
        set -- --target "$LLM_ENDPOINT" --target-type url --judge-model "$LLM_JUDGE_MODEL" \
               --format json,html,sarif --output-dir ./reports --fail-on-score "$LLM_FAIL_ON_SCORE"
        [ -n "${LLM_API_KEY:-}" ] && set -- "$@" --api-key "$LLM_API_KEY"
        [ -n "$LLM_SEVERITY" ] && set -- "$@" --severity "$LLM_SEVERITY"
        [ -n "$LLM_CATEGORIES" ] && set -- "$@" --categories "$LLM_CATEGORIES"
        [ "$LLM_INCLUDE_DOS_TESTS" = "true" ] && set -- "$@" --include-dos-tests
        llm-scanner "$@"
      # Add LLM_API_KEY only if the target requires bearer auth - kept out of a single interpolated string on purpose
  artifacts:
    when: always
    paths:
      - reports/

GitHub Actions (Docker)

# .github/workflows/llm-security.yml
name: LLM Security Scan
on: [pull_request, workflow_dispatch]

jobs:
  scan:
    runs-on: ubuntu-latest
    env:
      LLM_ENDPOINT: ${{ vars.LLM_ENDPOINT }}
      OLLAMA_HOST: ${{ vars.OLLAMA_HOST || 'http://host.docker.internal:11434' }}
      LLM_JUDGE_MODEL: ${{ vars.LLM_JUDGE_MODEL || 'llama3.2:3b' }}
      LLM_FAIL_ON_SCORE: ${{ vars.LLM_FAIL_ON_SCORE || '7.0' }}
      LLM_SCANNER_VERSION: ${{ vars.LLM_SCANNER_VERSION }}  # empty = latest release from PyPI
      LLM_API_KEY: ${{ secrets.LLM_API_KEY }}  # pass secrets via env:, never inline in a run: script
    steps:
      - name: Validate configuration
        run: test -n "$LLM_ENDPOINT"
      - name: Build scanner image
        run: |
          cat <<'EOF' > Dockerfile.llm-scanner
          FROM python:3.11-slim
          ARG SCANNER_VERSION=
          RUN pip install --no-cache-dir "llm-security-scanner${SCANNER_VERSION:+==$SCANNER_VERSION}"
          ENTRYPOINT ["llm-scanner"]
          EOF
          docker build --build-arg SCANNER_VERSION="$LLM_SCANNER_VERSION" -f Dockerfile.llm-scanner -t llm-security-scanner .
      - name: Run scan in Docker
        run: |
          mkdir -p reports
          ARGS=(--target "$LLM_ENDPOINT" --target-type url --judge-model "$LLM_JUDGE_MODEL"
                --format json,html,sarif --output-dir /reports --fail-on-score "$LLM_FAIL_ON_SCORE")
          [ -n "$LLM_API_KEY" ] && ARGS+=(--api-key "$LLM_API_KEY")
          docker run --rm \
            --add-host host.docker.internal:host-gateway \
            -e OLLAMA_HOST="$OLLAMA_HOST" \
            -v "$PWD/reports:/reports" \
            llm-security-scanner \
            "${ARGS[@]}"
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: llm-security-reports
          path: reports/

GitLab CI (Docker)

# .gitlab-ci.yml
llm-security-scan:
  stage: test
  image: docker:27
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "web"
  services:
    - name: docker:27-dind
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    LLM_ENDPOINT: ""  # no default on purpose - set as a CI/CD variable
    OLLAMA_HOST: ""  # no default on purpose - set as a CI/CD variable
    LLM_JUDGE_MODEL: llama3.2:3b
    LLM_FAIL_ON_SCORE: "7.0"
    LLM_SCANNER_VERSION: ""  # empty = latest release from PyPI
  before_script:
    - test -n "$LLM_ENDPOINT"
    - test -n "$OLLAMA_HOST"
    - |
        cat <<'EOF' > Dockerfile.llm-scanner
        FROM python:3.11-slim
        ARG SCANNER_VERSION=
        RUN pip install --no-cache-dir "llm-security-scanner${SCANNER_VERSION:+==$SCANNER_VERSION}"
        ENTRYPOINT ["llm-scanner"]
        EOF
        docker build --build-arg SCANNER_VERSION="$LLM_SCANNER_VERSION" -f Dockerfile.llm-scanner -t llm-security-scanner .
  script:
    - |
        mkdir -p reports
        docker run --rm \
          -e OLLAMA_HOST="$OLLAMA_HOST" \
          -v "$CI_PROJECT_DIR/reports:/reports" \
          llm-security-scanner \
          --target "$LLM_ENDPOINT" \
          --target-type url \
          --judge-model "$LLM_JUDGE_MODEL" \
          --format json,html,sarif \
          --output-dir /reports \
          --fail-on-score "$LLM_FAIL_ON_SCORE"
      # Add -e LLM_API_KEY="$LLM_API_KEY" and --api-key "$LLM_API_KEY" only if needed
  artifacts:
    when: always
    paths:
      - reports/