{
  "schemaVersion": 1,
  "published": true,
  "reportType": "preliminary",
  "updated": "2026-11-14",
  "title": "Three AI models on 40 visual reasoning questions",
  "summary": "An exploratory report on accuracy, completion, API latency and cost. One 40-question run per model, with separate development checks showing how reasoning limits and response delivery affected completion.",
  "url": "https://www.universaliqtest.com/ai-iq-test",
  "baseline": {
    "id": "full-2026-11-14-v3",
    "completedAt": "2026-11-14T13:05:48.514Z",
    "questionsPerModel": 40,
    "runsPerModel": 1,
    "reasoningEffort": "medium",
    "maxOutputTokens": 8192,
    "evaluatedQuestionSlots": 120,
    "apiRequests": 121,
    "apiCostUsd": 4.4780999999999995,
    "protocolFingerprint": "f4425b7fb5a235cd7c11fb84c4849c4f7df3d43064160f289b8434ec9478f59b",
    "scoringVersion": "iq-40-2026-09:key-v1",
    "transportFailures": 1,
    "transportRetries": 1,
    "transportFailureCostUsd": 0,
    "transportFailureDurationMs": 29516
  },
  "models": [
    {
      "name": "Gemini 4 Carbon",
      "vendor": "Google",
      "id": "google/gemini-4-carbon",
      "version": "google/gemini-4-carbon-20261110",
      "provider": "Google AI Studio",
      "questions": 40,
      "correctAnswers": 34,
      "wrongAnswers": 1,
      "cutoffs": 5,
      "validAnswers": 35,
      "accuracyPercent": 85,
      "apiCostUsd": 0.92182,
      "medianDurationMs": 13871.5,
      "minDurationMs": 4882,
      "maxDurationMs": 107946,
      "totalDurationMs": 1164469,
      "promptTokens": 53920,
      "completionTokens": 81398
    },
    {
      "name": "Claude Fable 5.2",
      "vendor": "Anthropic",
      "id": "anthropic/claude-fable-5.2",
      "version": "anthropic/claude-fable-5.2-20261109",
      "provider": "Anthropic",
      "questions": 40,
      "correctAnswers": 32,
      "wrongAnswers": 7,
      "cutoffs": 1,
      "validAnswers": 39,
      "accuracyPercent": 80,
      "apiCostUsd": 1.7596799999999995,
      "medianDurationMs": 3984.5,
      "minDurationMs": 2485,
      "maxDurationMs": 82278,
      "totalDurationMs": 404448,
      "promptTokens": 107800,
      "completionTokens": 22432
    },
    {
      "name": "GPT-6 Astra",
      "vendor": "OpenAI",
      "id": "openai/gpt-6-astra",
      "version": "openai/gpt-6-astra-20260903",
      "provider": "OpenAI",
      "questions": 40,
      "correctAnswers": 37,
      "wrongAnswers": 3,
      "cutoffs": 0,
      "validAnswers": 40,
      "accuracyPercent": 92.5,
      "apiCostUsd": 1.7965999999999995,
      "medianDurationMs": 3740.5,
      "minDurationMs": 2138,
      "maxDurationMs": 71790,
      "totalDurationMs": 451226,
      "promptTokens": 89920,
      "completionTokens": 13458
    }
  ],
  "calibration": {
    "scored": false,
    "selectedPositions": [
      24,
      28,
      35,
      36,
      38,
      40
    ],
    "selection": "Union of the positions with a token-limit failure for at least one model in the original run.",
    "profiles": [
      {
        "id": "calibration-2026-11-14-v4:openai/gpt-6-astra",
        "modelName": "GPT-6 Astra",
        "modelId": "openai/gpt-6-astra",
        "version": "openai/gpt-6-astra-20260903",
        "provider": "OpenAI",
        "reasoningEffort": "high",
        "maxOutputTokens": 32768,
        "reasoningDelivery": "excluded",
        "expected": 6,
        "validAnswers": 6,
        "cutoffs": 0,
        "unavailableSlots": 0,
        "maxCompletionTokens": 6749,
        "costUsd": 1.083305,
        "scored": false
      },
      {
        "id": "calibration-2026-11-14-v4:anthropic/claude-fable-5.2",
        "modelName": "Claude Fable 5.2",
        "modelId": "anthropic/claude-fable-5.2",
        "version": "anthropic/claude-fable-5.2-20261109",
        "provider": "Anthropic",
        "reasoningEffort": "high",
        "maxOutputTokens": 32768,
        "reasoningDelivery": "excluded",
        "expected": 6,
        "validAnswers": 6,
        "cutoffs": 0,
        "unavailableSlots": 0,
        "maxCompletionTokens": 5473,
        "costUsd": 0.80152,
        "scored": false
      },
      {
        "id": "calibration-2026-11-14-v4:google/gemini-4-carbon",
        "modelName": "Gemini 4 Carbon",
        "modelId": "google/gemini-4-carbon",
        "version": "google/gemini-4-carbon-20261110",
        "provider": "Google AI Studio",
        "reasoningEffort": "high",
        "maxOutputTokens": 32768,
        "reasoningDelivery": "excluded",
        "expected": 6,
        "validAnswers": 3,
        "cutoffs": 1,
        "unavailableSlots": 2,
        "maxCompletionTokens": 32768,
        "costUsd": 1.046414,
        "scored": false
      },
      {
        "id": "calibration-vertex-2026-11-14-v5",
        "modelName": "Gemini 4 Carbon",
        "modelId": "google/gemini-4-carbon",
        "version": "google/gemini-4-carbon-20261110",
        "provider": "Google Vertex global",
        "reasoningEffort": "high",
        "maxOutputTokens": 65536,
        "reasoningDelivery": "excluded",
        "expected": 6,
        "validAnswers": 3,
        "cutoffs": 0,
        "unavailableSlots": 3,
        "maxCompletionTokens": 17655,
        "costUsd": 0.49329799999999996,
        "scored": false
      },
      {
        "id": "calibration-progress-2026-11-14-v6",
        "modelName": "Gemini 4 Carbon",
        "modelId": "google/gemini-4-carbon",
        "version": "google/gemini-4-carbon-20261110",
        "provider": "Google Vertex global",
        "reasoningEffort": "high",
        "maxOutputTokens": 65536,
        "reasoningDelivery": "included_then_discarded",
        "expected": 6,
        "validAnswers": 6,
        "cutoffs": 0,
        "unavailableSlots": 0,
        "maxCompletionTokens": 51145,
        "costUsd": 1.4190260000000001,
        "scored": false
      }
    ],
    "apiCostUsd": 4.843563,
    "evaluatedQuestionSlots": 30,
    "apiRequests": 31,
    "unbilledTransportFailures": 6,
    "initialCarbonRecoveryCount": 1,
    "finalCarbonProfileId": "calibration-progress-2026-11-14-v6",
    "longResponse": {
      "questionPosition": 38,
      "completionTokens": 51145,
      "durationMs": 656051
    }
  },
  "technicalPilots": {
    "apiRequests": 12,
    "apiCostUsd": 0.334781,
    "usedForScores": false
  },
  "totalApiCostUsd": 9.656444,
  "feesIncluded": false,
  "downloads": {
    "results": "/studies/ai-visual-reasoning-2026-11-14/results.json",
    "measurements": "/studies/ai-visual-reasoning-2026-11-14/measurements.json",
    "methods": "/studies/ai-visual-reasoning-2026-11-14/methods.json",
    "report": "/studies/ai-visual-reasoning-2026-11-14/report.md",
    "figure": "/studies/ai-visual-reasoning-2026-11-14/outcomes.svg",
    "figurePng": "/studies/ai-visual-reasoning-2026-11-14/outcomes.png",
    "manifest": "/studies/ai-visual-reasoning-2026-11-14/manifest.json"
  },
  "citation": "UniversalIQTest.com (2026). Three AI models on 40 visual reasoning questions: preliminary report. 14 November 2026. https://www.universaliqtest.com/ai-iq-test",
  "sources": [
    {
      "title": "OpenRouter: reasoning tokens and output limits",
      "url": "https://openrouter.ai/docs/guides/best-practices/reasoning-tokens"
    },
    {
      "title": "OECD: human-oriented tests as measures of artificial intelligence",
      "url": "https://www.oecd.org/en/publications/ai-and-the-future-of-skills-volume-1_5ee71f34-en/full-report/using-human-skills-taxonomies-and-tests-as-measures-of-artificial-intelligence_06d3b428.html"
    }
  ]
}