{
  "scope": "Original synthetic parent/reply pairs. Recorded real API outputs, not a spam-detection accuracy benchmark.",
  "testedAt": "2026-09-27T07:15:32.505Z",
  "samples": [
    {
      "id": "question",
      "label": "Useful question",
      "request": {
        "context": "Parent: We are debugging repeated API timeouts.\nReply: Have you checked the upstream timeout and retry limit?",
        "question": "Classify reply against parent. Relevant discussion: a substantive question, answer, correction or disagreement about the parent topic, including a directly useful reference. Unsolicited promotion: clear unrelated commercial pitch or follow/engagement solicitation with no substantive contribution. Needs review: too little context, ambiguous intent or mixed meaningful contribution and promotion. Criticism alone is not spam. Judge supplied text only; do not infer posting history or follow links.",
        "choices": [
          "Relevant discussion",
          "Unsolicited promotion",
          "Needs review"
        ]
      },
      "observed": {
        "id": "question",
        "requestSha256": "aa259a18857351b3085f9d9dee55f939ceb1e02a5733dbb77ea46fc119cec8d7",
        "model": "jev-1.13.0",
        "choice": "Relevant discussion",
        "probabilities": {
          "Relevant discussion": 1,
          "Unsolicited promotion": 0,
          "Needs review": 0
        },
        "confidence": 1,
        "metrics": {
          "providerDurationMs": 1687,
          "inputTokens": 541,
          "outputTokens": 49,
          "estimatedCostUsd": 2.2722e-05,
          "costStatus": "estimated",
          "currency": "USD",
          "pricingModel": "jev-1.13.0"
        }
      }
    },
    {
      "id": "pitch",
      "label": "Unrelated pitch",
      "request": {
        "context": "Parent: We are debugging repeated API timeouts.\nReply: Buy my weight-loss package today! Limited offer, message me for prices.",
        "question": "Classify reply against parent. Relevant discussion: a substantive question, answer, correction or disagreement about the parent topic, including a directly useful reference. Unsolicited promotion: clear unrelated commercial pitch or follow/engagement solicitation with no substantive contribution. Needs review: too little context, ambiguous intent or mixed meaningful contribution and promotion. Criticism alone is not spam. Judge supplied text only; do not infer posting history or follow links.",
        "choices": [
          "Relevant discussion",
          "Unsolicited promotion",
          "Needs review"
        ]
      },
      "observed": {
        "id": "pitch",
        "requestSha256": "54c7f947c569088d3373ae997a44180d6f6f214ec65d02aed3f015a19617666f",
        "model": "jev-1.13.0",
        "choice": "Unsolicited promotion",
        "probabilities": {
          "Relevant discussion": 0,
          "Unsolicited promotion": 1,
          "Needs review": 0
        },
        "confidence": 1,
        "metrics": {
          "providerDurationMs": 911,
          "inputTokens": 546,
          "outputTokens": 50,
          "estimatedCostUsd": 2.2932e-05,
          "costStatus": "estimated",
          "currency": "USD",
          "pricingModel": "jev-1.13.0"
        }
      }
    },
    {
      "id": "disagree",
      "label": "Relevant disagreement",
      "request": {
        "context": "Parent: We are debugging repeated API timeouts.\nReply: I disagree with adding more retries. They can amplify overload; try exponential backoff.",
        "question": "Classify reply against parent. Relevant discussion: a substantive question, answer, correction or disagreement about the parent topic, including a directly useful reference. Unsolicited promotion: clear unrelated commercial pitch or follow/engagement solicitation with no substantive contribution. Needs review: too little context, ambiguous intent or mixed meaningful contribution and promotion. Criticism alone is not spam. Judge supplied text only; do not infer posting history or follow links.",
        "choices": [
          "Relevant discussion",
          "Unsolicited promotion",
          "Needs review"
        ]
      },
      "observed": {
        "id": "disagree",
        "requestSha256": "c27e79c12f376f08fbc4b0efeda2be69d33169b3aa821f8e06e8d745977da48a",
        "model": "jev-1.13.0",
        "choice": "Relevant discussion",
        "probabilities": {
          "Relevant discussion": 1,
          "Unsolicited promotion": 0,
          "Needs review": 0
        },
        "confidence": 1,
        "metrics": {
          "providerDurationMs": 1038,
          "inputTokens": 548,
          "outputTokens": 49,
          "estimatedCostUsd": 2.3016e-05,
          "costStatus": "estimated",
          "currency": "USD",
          "pricingModel": "jev-1.13.0"
        }
      }
    },
    {
      "id": "follow",
      "label": "Follow solicitation",
      "request": {
        "context": "Parent: We are debugging repeated API timeouts.\nReply: Follow me and I will follow you back!",
        "question": "Classify reply against parent. Relevant discussion: a substantive question, answer, correction or disagreement about the parent topic, including a directly useful reference. Unsolicited promotion: clear unrelated commercial pitch or follow/engagement solicitation with no substantive contribution. Needs review: too little context, ambiguous intent or mixed meaningful contribution and promotion. Criticism alone is not spam. Judge supplied text only; do not infer posting history or follow links.",
        "choices": [
          "Relevant discussion",
          "Unsolicited promotion",
          "Needs review"
        ]
      },
      "observed": {
        "id": "follow",
        "requestSha256": "483803f735425381ffe39f73becfe67a1a9929480151ba664dc37312e9a6073a",
        "model": "jev-1.13.0",
        "choice": "Unsolicited promotion",
        "probabilities": {
          "Relevant discussion": 0,
          "Unsolicited promotion": 1,
          "Needs review": 0
        },
        "confidence": 1,
        "metrics": {
          "providerDurationMs": 1045,
          "inputTokens": 540,
          "outputTokens": 50,
          "estimatedCostUsd": 2.268e-05,
          "costStatus": "estimated",
          "currency": "USD",
          "pricingModel": "jev-1.13.0"
        }
      }
    },
    {
      "id": "mixed",
      "label": "Advice plus promotion",
      "request": {
        "context": "Parent: We are debugging repeated API timeouts.\nReply: Set an explicit request timeout; also buy my consulting package to get it working.",
        "question": "Classify reply against parent. Relevant discussion: a substantive question, answer, correction or disagreement about the parent topic, including a directly useful reference. Unsolicited promotion: clear unrelated commercial pitch or follow/engagement solicitation with no substantive contribution. Needs review: too little context, ambiguous intent or mixed meaningful contribution and promotion. Criticism alone is not spam. Judge supplied text only; do not infer posting history or follow links.",
        "choices": [
          "Relevant discussion",
          "Unsolicited promotion",
          "Needs review"
        ]
      },
      "observed": {
        "id": "mixed",
        "requestSha256": "4ddcf8de5ec35b32aaa904df386f217a4c4e885f4d629aec4bfd33568e201e8f",
        "model": "jev-1.13.0",
        "choice": "Needs review",
        "probabilities": {
          "Relevant discussion": 0.05,
          "Unsolicited promotion": 0.07,
          "Needs review": 0.88
        },
        "confidence": 0.82,
        "metrics": {
          "providerDurationMs": 1045,
          "inputTokens": 547,
          "outputTokens": 47,
          "estimatedCostUsd": 2.2974e-05,
          "costStatus": "estimated",
          "currency": "USD",
          "pricingModel": "jev-1.13.0"
        }
      }
    },
    {
      "id": "missing",
      "label": "Missing parent",
      "request": {
        "context": "Parent: Not supplied.\nReply: This might help; take a look.",
        "question": "Classify reply against parent. Relevant discussion: a substantive question, answer, correction or disagreement about the parent topic, including a directly useful reference. Unsolicited promotion: clear unrelated commercial pitch or follow/engagement solicitation with no substantive contribution. Needs review: too little context, ambiguous intent or mixed meaningful contribution and promotion. Criticism alone is not spam. Judge supplied text only; do not infer posting history or follow links.",
        "choices": [
          "Relevant discussion",
          "Unsolicited promotion",
          "Needs review"
        ]
      },
      "observed": {
        "id": "missing",
        "requestSha256": "6c93032f027cf37b0dd7178205f81beb3eec81bd59091165cf86f3a148409fd2",
        "model": "jev-1.13.0",
        "choice": "Needs review",
        "probabilities": {
          "Relevant discussion": 0.05,
          "Unsolicited promotion": 0,
          "Needs review": 0.95
        },
        "confidence": 0.91,
        "metrics": {
          "providerDurationMs": 1251,
          "inputTokens": 534,
          "outputTokens": 47,
          "estimatedCostUsd": 2.2428e-05,
          "costStatus": "estimated",
          "currency": "USD",
          "pricingModel": "jev-1.13.0"
        }
      }
    }
  ]
}
