{"id":"wshobson-agents-plugins-llm-finetuning-skills-eval-harness-first","title":"eval-harness-first","type":"skill","industry":"data-research","roles":["general"],"summary":"Build the evaluation harness that gates every fine-tuning run — golden sets, per-failure-mode graders, judge calibration, and base-model baselines. Use when starting a fine-tuning effort, when converting traces into an eval set, or when calibrating a judge against human labels.","summary_lang":"en","source_url":"https://github.com/wshobson/agents/blob/main/plugins/llm-finetuning/skills/eval-harness-first/SKILL.md","raw_url":"https://raw.githubusercontent.com/SiinXu/agent-academy/main/data/mirror/wshobson-agents-plugins-llm-finetuning-skills-eval-harness-first.md","upstream_raw_url":"https://raw.githubusercontent.com/wshobson/agents/main/plugins/llm-finetuning/skills/eval-harness-first/SKILL.md","repo":"wshobson/agents","author":"wshobson","license":"MIT","redistributable":true,"content_policy":"mirror-with-attribution","original":{"kind":"full","license":"MIT","url":"https://github.com/wshobson/agents/blob/main/plugins/llm-finetuning/skills/eval-harness-first/SKILL.md"},"install":"npx skills add wshobson/agents --skill eval-harness-first","security":{"audited":false,"executes_code":false,"needs_credentials":false,"writes_external":false},"metrics":{"repo_stars":40824,"repo_pushed_at":"2026-10-14"},"course_step":"3-learn-rules","status":"active","first_seen":"2026-10-15T10:15:30Z","updated_at":"2026-10-15T10:15:30Z","checked_at":"2026-10-15T10:15:30Z","http_status":{"source":200},"page":"https://academy.incidental.site/r/wshobson-agents-plugins-llm-finetuning-skills-eval-harness-first/","source":"github"}