{
  "kind": "original-bounded-choice-workflow",
  "version": 1,
  "scope": "Choose which explicitly offered evaluation metric directly addresses a supplied evaluation goal: label agreement, distribution match or observed response time. Preserve a missing or compound goal; emit a pending metric-selection report without computing a metric or ranking models.",
  "question": "Select the offered metric that directly measures evaluationGoal. Label agreement: comparing chosen labels with independent reference labels. Distribution match: comparing the complete predicted probability distribution with the stated reference distribution. Observed response time: measuring elapsed request-to-response duration. Clarify goal: the goal is absent, requests multiple measures without priority, or is unsupported by these metrics. Do not compute a value or rank models.",
  "choices": [
    "Label agreement",
    "Distribution match",
    "Observed response time",
    "Clarify goal"
  ],
  "actions": {
    "Label agreement": {
      "action": "preview_metric_goal_selection",
      "payload": {
        "decision": "Label agreement"
      },
      "pending": true
    },
    "Distribution match": {
      "action": "preview_metric_goal_selection",
      "payload": {
        "decision": "Distribution match"
      },
      "pending": true
    },
    "Observed response time": {
      "action": "preview_metric_goal_selection",
      "payload": {
        "decision": "Observed response time"
      },
      "pending": true
    },
    "Clarify goal": {
      "action": "preview_metric_goal_selection",
      "payload": {
        "decision": "Clarify goal"
      },
      "pending": true
    }
  },
  "samples": [
    {
      "id": "labels",
      "label": "Agreement with independent labels",
      "expected": "Label agreement",
      "request": {
        "context": "{\n  \"task\": \"Review this metric selection goal.\",\n  \"evaluationGoal\": \"Measure how often chosen labels equal independently assigned reference labels.\"\n}",
        "question": "Select the offered metric that directly measures evaluationGoal. Label agreement: comparing chosen labels with independent reference labels. Distribution match: comparing the complete predicted probability distribution with the stated reference distribution. Observed response time: measuring elapsed request-to-response duration. Clarify goal: the goal is absent, requests multiple measures without priority, or is unsupported by these metrics. Do not compute a value or rank models.",
        "choices": [
          "Label agreement",
          "Distribution match",
          "Observed response time",
          "Clarify goal"
        ]
      }
    },
    {
      "id": "distribution",
      "label": "Match the complete reference distribution",
      "expected": "Distribution match",
      "request": {
        "context": "{\n  \"task\": \"Review this metric selection goal.\",\n  \"evaluationGoal\": \"Measure mismatch between each complete predicted probability distribution and the stated reference distribution.\"\n}",
        "question": "Select the offered metric that directly measures evaluationGoal. Label agreement: comparing chosen labels with independent reference labels. Distribution match: comparing the complete predicted probability distribution with the stated reference distribution. Observed response time: measuring elapsed request-to-response duration. Clarify goal: the goal is absent, requests multiple measures without priority, or is unsupported by these metrics. Do not compute a value or rank models.",
        "choices": [
          "Label agreement",
          "Distribution match",
          "Observed response time",
          "Clarify goal"
        ]
      }
    },
    {
      "id": "time",
      "label": "Elapsed request duration",
      "expected": "Observed response time",
      "request": {
        "context": "{\n  \"task\": \"Review this metric selection goal.\",\n  \"evaluationGoal\": \"Measure elapsed time from dispatching a request to receiving its response.\"\n}",
        "question": "Select the offered metric that directly measures evaluationGoal. Label agreement: comparing chosen labels with independent reference labels. Distribution match: comparing the complete predicted probability distribution with the stated reference distribution. Observed response time: measuring elapsed request-to-response duration. Clarify goal: the goal is absent, requests multiple measures without priority, or is unsupported by these metrics. Do not compute a value or rank models.",
        "choices": [
          "Label agreement",
          "Distribution match",
          "Observed response time",
          "Clarify goal"
        ]
      }
    },
    {
      "id": "missing",
      "label": "Goal absent",
      "expected": "Clarify goal",
      "request": {
        "context": "{\n  \"task\": \"Review this metric selection goal.\",\n  \"evaluationGoal\": null\n}",
        "question": "Select the offered metric that directly measures evaluationGoal. Label agreement: comparing chosen labels with independent reference labels. Distribution match: comparing the complete predicted probability distribution with the stated reference distribution. Observed response time: measuring elapsed request-to-response duration. Clarify goal: the goal is absent, requests multiple measures without priority, or is unsupported by these metrics. Do not compute a value or rank models.",
        "choices": [
          "Label agreement",
          "Distribution match",
          "Observed response time",
          "Clarify goal"
        ]
      }
    },
    {
      "id": "compound",
      "label": "Several goals without a priority",
      "expected": "Clarify goal",
      "request": {
        "context": "{\n  \"task\": \"Review this metric selection goal.\",\n  \"evaluationGoal\": \"Measure both label correctness and response duration; no priority or combined protocol has been chosen.\"\n}",
        "question": "Select the offered metric that directly measures evaluationGoal. Label agreement: comparing chosen labels with independent reference labels. Distribution match: comparing the complete predicted probability distribution with the stated reference distribution. Observed response time: measuring elapsed request-to-response duration. Clarify goal: the goal is absent, requests multiple measures without priority, or is unsupported by these metrics. Do not compute a value or rank models.",
        "choices": [
          "Label agreement",
          "Distribution match",
          "Observed response time",
          "Clarify goal"
        ]
      }
    },
    {
      "id": "unsupported",
      "label": "Billing audit outside the offered metrics",
      "expected": "Clarify goal",
      "request": {
        "context": "{\n  \"task\": \"Review this metric selection goal.\",\n  \"evaluationGoal\": \"Audit whether the invoice matches all charged provider operations.\"\n}",
        "question": "Select the offered metric that directly measures evaluationGoal. Label agreement: comparing chosen labels with independent reference labels. Distribution match: comparing the complete predicted probability distribution with the stated reference distribution. Observed response time: measuring elapsed request-to-response duration. Clarify goal: the goal is absent, requests multiple measures without priority, or is unsupported by these metrics. Do not compute a value or rank models.",
        "choices": [
          "Label agreement",
          "Distribution match",
          "Observed response time",
          "Clarify goal"
        ]
      }
    }
  ],
  "consumer": "preview_choice_workflow_v2.py"
}
