# Run evaluation (primary command)
lm-eval run \
  --model vllm \
  --model_args pretrained=meta-llama/Llama-3.1-8B-Instruct,tensor_parallel_size=2 \
  --tasks mmlu,gsm8k,hellaswag \
  --num_fewshot 5 \
  --batch_size auto \
  --apply_chat_template \
  --log_samples \
  --output_path ./results/

# Load all args from YAML config file (v0.4.11+)
lm-eval run --config eval_config.yaml

# Discover available tasks
lm-eval ls tasks                        # list all task names
lm-eval ls tasks --json | jq '.[].name' # machine-readable

# Validate task configs before running
lm-eval validate --tasks mmlu,gsm8k     # catch YAML errors early
lm-eval validate --tasks my_custom_task --include_path ./tasks/

# eval_config.yaml
model: vllm
model_args: pretrained=meta-llama/Llama-3.1-8B-Instruct,tensor_parallel_size=2,gpu_memory_utilization=0.9
tasks:
  - mmlu
  - gsm8k
  - hellaswag
num_fewshot: 5
batch_size: auto
apply_chat_template: true
log_samples: true
output_path: ./results/
seed: [0, 1234, 1234, 1234]

lm-eval run --config eval_config.yaml   # all flags from file; CLI overrides file
lm-eval run --config eval_config.yaml --limit 100  # override: only 100 samples

import lm_eval

results = lm_eval.simple_evaluate(
    model="vllm",
    model_args="pretrained=meta-llama/Llama-3.1-8B-Instruct,dtype=auto,tensor_parallel_size=2,gpu_memory_utilization=0.9",
    tasks=["mmlu", "gsm8k", "hellaswag"],
    num_fewshot=5,
    batch_size="auto",
    log_samples=True,
    apply_chat_template=True,
)

for task_name, task_result in results["results"].items():
    print(f"{task_name}: {task_result}")

generation_kwargs:
  max_gen_toks: 1024          # max tokens to generate
  temperature: 0.0            # 0.0 = greedy
  top_p: 1.0
  do_sample: false
  stop_sequences: ["\n\n"]    # stop on these strings
  until: ["\n\nQuestion:"]    # alternative stop format

filter_list:
  - name: get-answer
    filter:
      - function: regex
        regex_pattern: "#### (\\d+)"   # extract final answer
      - function: take_first
  - name: remove-whitespace
    filter:
      - function: strip
      - function: lowercase

results = lm_eval.simple_evaluate(
    model="vllm",
    model_args="pretrained=my-model",
    tasks=["mmlu"],
    wandb_args="project=eval,name=llama-8b-mmlu,job_type=eval",
)

results = lm_eval.simple_evaluate(
    model="vllm",
    model_args="pretrained=my-model,tensor_parallel_size=4",
    tasks=["mmlu_pro", "gpqa_main_zeroshot", "musr", "minerva_math_hard", "ifeval", "bbh"],
    batch_size="auto",
    apply_chat_template=True,
)

task: my_custom_task
dataset_path: json                    # or huggingface dataset name
dataset_name: null
dataset_kwargs:
  data_files:
    test: ./data/test.jsonl
output_type: generate_until           # or multiple_choice, loglikelihood
doc_to_text: "Question: {{question}}\nAnswer:"
doc_to_target: "{{answer}}"
generation_kwargs:
  max_gen_toks: 256
  temperature: 0.0
  do_sample: false
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
filter_list:
  - name: get-answer
    filter:
      - function: regex
        regex_pattern: "(\\d+)"
      - function: take_first
num_fewshot: 0

group: my_eval_suite
task:
  - my_custom_task
  - mmlu
  - gsm8k
aggregate_metric_list:
  - metric: acc
    aggregation: mean
    weight_by_size: true

include: _default_template.yaml
task: my_variant
dataset_name: hard_subset
num_fewshot: 0

# judge_quality.yaml — custom task that scores responses via LLM judge
task: judge_quality
dataset_path: json
dataset_kwargs:
  data_files:
    test: ./data/judge_test.jsonl  # {"question": "...", "reference": "..."}
output_type: generate_until
doc_to_text: "{{question}}"
doc_to_target: "{{reference}}"
generation_kwargs:
  max_gen_toks: 512
  temperature: 0.0
  do_sample: false
metric_list:
  - metric: judge_score
    aggregation: mean
    higher_is_better: true
process_results: !function utils.judge_score
metadata:
  version: 1.0

# utils.py — placed alongside judge_quality.yaml
import openai

# Point at your local vLLM (or any OpenAI-compatible) endpoint
_client = openai.OpenAI(
    base_url="http://vllm-svc:8000/v1",  # local vLLM service
    api_key="unused",                      # vLLM doesn't need a real key
)
_JUDGE_MODEL = "meta-llama/Llama-3.1-70B-Instruct"  # whatever model vLLM is serving

JUDGE_PROMPT = """Rate the response on a scale of 1-5 for accuracy and completeness.
Return ONLY a JSON object: {{"score": <int>, "reason": "<one sentence>"}}

Question: {question}
Reference: {reference}
Response: {response}"""

def judge_score(doc, results):
    """Called by the harness via process_results: !function utils.judge_score"""
    response_text = results[0]
    msg = JUDGE_PROMPT.format(
        question=doc["question"],
        reference=doc.get("reference", "N/A"),
        response=response_text,
    )
    completion = _client.chat.completions.create(
        model=_JUDGE_MODEL,
        messages=[{"role": "user", "content": msg}],
        temperature=0.0,
        response_format={"type": "json_object"},  # structured output — no fragile parsing
    )
    import json
    parsed = json.loads(completion.choices[0].message.content)
    return {"judge_score": int(parsed["score"])}

# Evaluate a model and score its outputs with the local vLLM judge
lm-eval run \
  --model vllm \
  --model_args pretrained=my-org/my-model,tensor_parallel_size=2 \
  --tasks judge_quality \
  --include_path ./tasks/ \
  --log_samples \
  --output_path ./results/

# Judge scores appear in standard results alongside any other metrics
# results["results"]["judge_quality"]["judge_score"] = 3.72

results = lm_eval.simple_evaluate(
    model="vllm",
    model_args="pretrained=my-org/my-model",
    tasks=["mmlu", "judge_quality"],
    include_path="./tasks/",
    log_samples=True,
    wandb_args="project=eval,name=my-model-judge",
)
# results["results"]["judge_quality"]["judge_score"] logged automatically to W&B