{ "schemaVersion": 1, "study": "reference-access-2026-09-25", "measuredDate": "2026-09-25", "model": "zai-org/GLM-5.3-Flash", "taskCount": 20, "agreementsPerTask": 50, "repetitions": 5, "sourceSegments": 3, "conditions": { "reference": { "attempts": 100, "correct": 99, "score": 0.99, "rowAccuracy": 0.99, "meanTurns": 2.03, "meanSeconds": 42.72062809954, "meanUsd": null, "knownUsd": "0.42709952", "missingUsageRequests": 1, "toolRejections": 0, "timeouts": 1, "incorrectAnswers": 0 }, "sql": { "attempts": 100, "correct": 99, "score": 0.99, "rowAccuracy": 0.9998, "meanTurns": 3.9, "meanSeconds": 86.32905955584, "meanUsd": "0.0092795195", "knownUsd": "0.92795195", "missingUsageRequests": 0, "toolRejections": 13, "timeouts": 0, "incorrectAnswers": 1 } }, "tasks": [ { "task": 1, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.2, "meanSeconds": 38.327311108, "meanUsd": "0.00430653", "knownUsd": "0.02153265", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 4.8, "meanSeconds": 114.4823609168, "meanUsd": "0.011750488", "knownUsd": "0.05875244", "missingUsageRequests": 0, "toolRejections": 1, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 2, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 34.4887552084, "meanUsd": "0.003738348", "knownUsd": "0.01869174", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.4, "meanSeconds": 68.0488149586, "meanUsd": "0.008012326", "knownUsd": "0.04006163", "missingUsageRequests": 0, "toolRejections": 1, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 3, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.2, "meanSeconds": 53.931124658, "meanUsd": "0.005354302", "knownUsd": "0.02677151", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 4.6, "meanSeconds": 83.3000624168, "meanUsd": "0.010442368", "knownUsd": "0.05221184", "missingUsageRequests": 0, "toolRejections": 1, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 4, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 42.8268248584, "meanUsd": "0.004876444", "knownUsd": "0.02438222", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.8, "meanSeconds": 89.72767485840001, "meanUsd": "0.00899481", "knownUsd": "0.04497405", "missingUsageRequests": 0, "toolRejections": 2, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 5, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 41.9050882748, "meanUsd": "0.003936276", "knownUsd": "0.01968138", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.2, "meanSeconds": 76.7795235084, "meanUsd": "0.008089686", "knownUsd": "0.04044843", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 6, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 30.8809026166, "meanUsd": "0.003731044", "knownUsd": "0.01865522", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.8, "meanSeconds": 92.49481775, "meanUsd": "0.008909814", "knownUsd": "0.04454907", "missingUsageRequests": 0, "toolRejections": 2, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 7, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 39.0626919752, "meanUsd": "0.004275966", "knownUsd": "0.02137983", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.8, "meanSeconds": 67.5194951668, "meanUsd": "0.007508184", "knownUsd": "0.03754092", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 8, "reference": { "attempts": 5, "correct": 4, "score": 0.8, "rowAccuracy": 0.8, "meanTurns": 2.0, "meanSeconds": 86.797205658, "meanUsd": null, "knownUsd": "0.01993059", "missingUsageRequests": 1, "toolRejections": 0, "timeouts": 1, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.4, "meanSeconds": 68.2971895418, "meanUsd": "0.007709522", "knownUsd": "0.03854761", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 9, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 33.4962429082, "meanUsd": "0.003416362", "knownUsd": "0.01708181", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 4, "score": 0.8, "rowAccuracy": 0.996, "meanTurns": 4.0, "meanSeconds": 86.9433984168, "meanUsd": "0.009076082", "knownUsd": "0.04538041", "missingUsageRequests": 0, "toolRejections": 1, "timeouts": 0, "incorrectAnswers": 1 } }, { "task": 10, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 55.467203058399996, "meanUsd": "0.005437884", "knownUsd": "0.02718942", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 5.0, "meanSeconds": 124.619440575, "meanUsd": "0.014031316", "knownUsd": "0.07015658", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 11, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 51.527233366800004, "meanUsd": "0.004284808", "knownUsd": "0.02142404", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.0, "meanSeconds": 96.46759979160001, "meanUsd": "0.008534684", "knownUsd": "0.04267342", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 12, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 36.1677544666, "meanUsd": "0.004063918", "knownUsd": "0.02031959", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.2, "meanSeconds": 76.8066393164, "meanUsd": "0.008029152", "knownUsd": "0.04014576", "missingUsageRequests": 0, "toolRejections": 1, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 13, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 47.7420702918, "meanUsd": "0.004867722", "knownUsd": "0.02433861", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 4.2, "meanSeconds": 82.234858625, "meanUsd": "0.009425594", "knownUsd": "0.04712797", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 14, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 34.592157433400004, "meanUsd": "0.004063198", "knownUsd": "0.02031599", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 4.4, "meanSeconds": 75.1836007584, "meanUsd": "0.010267858", "knownUsd": "0.05133929", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 15, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 37.7454683332, "meanUsd": "0.003836038", "knownUsd": "0.01918019", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 5.0, "meanSeconds": 82.0651075168, "meanUsd": "0.009495098", "knownUsd": "0.04747549", "missingUsageRequests": 0, "toolRejections": 4, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 16, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 36.55686595, "meanUsd": "0.003842888", "knownUsd": "0.01921444", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.8, "meanSeconds": 83.1767726914, "meanUsd": "0.009442694", "knownUsd": "0.04721347", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 17, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 34.6938161916, "meanUsd": "0.003892858", "knownUsd": "0.01946429", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.6, "meanSeconds": 93.7638201164, "meanUsd": "0.008441688", "knownUsd": "0.04220844", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 18, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.2, "meanSeconds": 44.374911158, "meanUsd": "0.00488219", "knownUsd": "0.02441095", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 4.2, "meanSeconds": 79.2220699084, "meanUsd": "0.009382056", "knownUsd": "0.04691028", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 19, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 32.361928067, "meanUsd": "0.00438851", "knownUsd": "0.02194255", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 3.8, "meanSeconds": 96.6630610664, "meanUsd": "0.009701564", "knownUsd": "0.04850782", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } }, { "task": 20, "reference": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 2.0, "meanSeconds": 41.467006408399996, "meanUsd": "0.00423850", "knownUsd": "0.02119250", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 }, "sql": { "attempts": 5, "correct": 5, "score": 1.0, "rowAccuracy": 1.0, "meanTurns": 4.0, "meanSeconds": 88.7848832166, "meanUsd": "0.008345406", "knownUsd": "0.04172703", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 0 } } ], "sensitivity": { "rejectedCalls": 13, "affectedPairs": 12, "affectedTasks": 8, "allPairs": { "pairs": 100, "reference": { "attempts": 100, "correct": 99, "score": 0.99, "rowAccuracy": 0.99, "meanTurns": 2.03, "meanSeconds": 42.72062809954, "meanUsd": null, "knownUsd": "0.42709952", "missingUsageRequests": 1, "toolRejections": 0, "timeouts": 1, "incorrectAnswers": 0 }, "sql": { "attempts": 100, "correct": 99, "score": 0.99, "rowAccuracy": 0.9998, "meanTurns": 3.9, "meanSeconds": 86.32905955584, "meanUsd": "0.0092795195", "knownUsd": "0.92795195", "missingUsageRequests": 0, "toolRejections": 13, "timeouts": 0, "incorrectAnswers": 1 }, "turnReductionFraction": 0.47948717948717956 }, "withoutRejectedSqlPairs": { "pairs": 88, "reference": { "attempts": 88, "correct": 87, "score": 0.9886363636363636, "rowAccuracy": 0.9886363636363636, "meanTurns": 2.034090909090909, "meanSeconds": 43.558091711125, "meanUsd": null, "knownUsd": "0.38031349", "missingUsageRequests": 1, "toolRejections": 0, "timeouts": 1, "incorrectAnswers": 0 }, "sql": { "attempts": 88, "correct": 87, "score": 0.9886363636363636, "rowAccuracy": 0.9997727272727273, "meanTurns": 3.6818181818181817, "meanSeconds": 83.86553571306818, "meanUsd": "0.009017993068181818181818181818", "knownUsd": "0.79358339", "missingUsageRequests": 0, "toolRejections": 0, "timeouts": 0, "incorrectAnswers": 1 }, "turnReductionFraction": 0.44753086419753085 }, "postHoc": true, "causalEffectOfRemovingGuardEstablished": false }, "pairedAnalysis": { "method": "paired_task_cluster_percentile_bootstrap", "independent_task_groups": 20, "draws": 20000, "seed": 1701, "difference_direction": "reference_minus_sql", "metrics": { "model_turns": { "reference_minus_sql": -1.8699999999999997, "percentile_95_interval": [ -2.1300000000000003, -1.6100000000000003 ], "no_observed_task_variation_in_difference": false }, "latency_seconds": { "reference_minus_sql": -43.6084314563, "percentile_95_interval": [ -51.597878808387755, -34.45088028819126 ], "no_observed_task_variation_in_difference": false }, "exact_table_accuracy": { "reference_minus_sql": 0.0, "percentile_95_interval": [ -0.030000000000000027, 0.030000000000000027 ], "no_observed_task_variation_in_difference": false }, "row_accuracy": { "reference_minus_sql": -0.009800000000000031, "percentile_95_interval": [ -0.030000000000000027, 0.0006000000000000449 ], "no_observed_task_variation_in_difference": false } }, "cost_comparison_available": false, "boundary_caution": "A degenerate accuracy interval does not establish equivalence or rule out unseen failures." }, "excludedProviderRejections": 1, "protocol": { "reasoning": "max", "temperature": 0.2, "maxModelRequests": 12, "maxOutputTokens": 16384, "timeoutSeconds": 240, "score": "exact_50_row_table", "cost": "received_token_usage_at_published_rates" } }