{ "arms": { "base_hermes": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 60, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 1926022, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 19872, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 60, "observed_total": 1945078.149, "total": 1945078.149 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 352, "total": null } }, "attributed_completed": 49, "completion_percent": 81.66666666666667, "observed_completed": 49, "outcomes": { "fail": 11, "pass": 49 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 60, "max": 86520.422, "median": 24717.3265, "min": 14596.628, "samples": 60, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 345, "max": 15552.917, "median": 1974.627, "min": 1728.711, "samples": 85, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 345, "max": 22849.341, "median": 1891.12, "min": 1679.827, "samples": 344, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 352, "max": 23046.879, "median": 2432.4449999999997, "min": 1738.332, "samples": 352, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 352, "max": 68.02591714297, "median": 14.883890675593427, "min": 0.8299171867135238, "samples": 352, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 352, "instrumented": 352, "observed": 352, "streaming": 345, "with_first_content": 85, "with_first_generated": 344, "with_usage": 352 } } }, "protagine": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 60, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 2576028, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 39953, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 60, "observed_total": 2286629.6520000002, "total": 2286629.6520000002 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 537, "total": null } }, "attributed_completed": 47, "completion_percent": 78.33333333333333, "observed_completed": 47, "outcomes": { "fail": 13, "pass": 47 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 60, "max": 105339.474, "median": 33398.2745, "min": 19621.191, "samples": 60, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 315, "max": 20136.705, "median": 2717.8985000000002, "min": 2071.392, "samples": 88, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 315, "max": 20169.982, "median": 2472.65, "min": 1956.604, "samples": 313, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 483, "max": 26643.07, "median": 3188.624, "min": 1002.452, "samples": 483, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 483, "max": 75.97975024535127, "median": 17.871196785933993, "min": 1.0379456719773805, "samples": 483, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 483, "instrumented": 537, "observed": 537, "streaming": 315, "with_first_content": 88, "with_first_generated": 313, "with_usage": 483 } } } }, "budget_mode": "deployment_policy", "budget_verification": "unverified", "comparable_pairs": 60, "comparison_key": "94d3846a77535d69661893323da9297a4150c0669205d67b47eb9ff393b42b8e", "completion_projection": { "artifact_verifier_reexecuted": false, "basis": "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun.", "corrected_episodes": 1, "raw_results_preserved": true, "rule": "returned_native_no_output_noncompletion", "source_protocol": "paired-attribution-2" }, "dataset": { "sha256": "9d513d5288be1ef6043a930c66ff1513e810c7fcefb7a58576668e5c804e7127", "source_sha256": "888e6982b9239dd76b35a2f26feb851172ed070c9f932eab1bef431953e09a74", "split": "development", "version": "paired-agent-reviewed-2" }, "declared_episodes": 60, "deployment": { "concurrency": 4, "context_limit": 327680, "hardware": "Two NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation GPUs, 96 GB per GPU", "id": "glm53-flash-abliterated-rtx-tp2-high", "model": "GLM-5.3 Flash Abliterated", "node_count": 1, "profile": "Frozen text-agent baseline; 4096 output tokens per call; 8 iterations; isolated state per attempt", "quantization": "EXL3 4 bpw weights; NVFP4 MLA KV cache", "revision": "c8f58e6aa9117c73607d692978b22f091d80450c", "runtime": "vLLM", "runtime_version": "0.26.1rc0+infernal.invocation.cu133.r19.vllm174c789.b12x12c4263", "source_url": "https://huggingface.co/lovesenko/GLM-5.3-Flash-tr3-4bpw-Abliterated", "variant": "abliterated", "weights_verified": false }, "endpoint_usage": "shared", "episodes": [ { "comparison": "tie", "episode_id": "paired.grounded-evidence.dispatch-is-not-delivery", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15824.849, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 22790.017, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.inventory-event-deduplication", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15823.0, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25495.216, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.unequal-measurement-denominators", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16426.834, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25608.073, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.shared-resource-schedule", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16122.678, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 37090.213, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.stale-handoff-reference", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16012.415, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 29005.521, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.planning-toolrecovery.approval-expiry-at-planning-time", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 29335.189, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 27605.963, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.scoped-correction-in-new-session", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 49871.996, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 46304.059, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.separate-peoples-preferences", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 53024.002, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 47915.794, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.revoked-transient-detail", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 71593.507, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 43330.696, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.crosssession-authority.sequential-task-handoff", "family": "crosssession-authority", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 59699.467, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 56394.561, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.crosssession-authority.public-artifact-disclosure-scope", "family": "crosssession-authority", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 42611.223, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 59446.415, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.instructions-inside-untrusted-report", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17961.596, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 20908.29, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.python-syntax-repair", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 22276.903, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 34323.318, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.bounded-retry-configuration", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 20557.905, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 28078.297, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.environment-to-typed-config", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 14596.628, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 20986.688, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.extraction-review.quotation-is-not-personal-history", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 38820.462, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": { "rule": "returned_native_no_output_noncompletion", "source_row_sha256": "64305d220778add4724e15ecbb558ccea44500f406c4934ebeec298597259ebe" }, "elapsed_ms": 34941.723, "outcome": "fail", "primary_outcome": "unverified" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.condition-is-not-authorization", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17768.172, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 19621.191, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.review-claims-against-measurements", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19955.157, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 24009.885, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.observation-time-beats-file-order", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15047.263, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 20212.31, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.grounded-evidence.equally-supported-conflict", "family": "grounded-evidence", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 40037.969, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 28308.716, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.missing-observations-are-not-zero", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15988.475, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 28182.469, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.settled-refunds-and-pending-reversals", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16238.228, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 37681.865, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.correlation-with-confounded-change", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17536.171, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 26721.618, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.binary-capacity-and-decimal-sizes", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 31222.328, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25582.16, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.grounded-evidence.half-open-observation-window", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 33391.338, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 45348.873, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.independent-resources-critical-path", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 43895.289, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25960.853, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.idempotent-resume-plan", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15175.562, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25401.97, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.bounded-selection-with-blocked-work", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 23496.051, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 37471.841, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.planning-toolrecovery.timeout-reconciliation-before-retry", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 21022.165, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 26547.448, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.file-restoration-after-failed-check", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 27939.461, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 38639.98, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.planning-toolrecovery.consent-does-not-expand-action-scope", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 33027.295, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 50746.481, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.dependency-cycle-with-independent-work", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19838.821, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 20553.369, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.persistent-memory.one-day-exception-does-not-change-default", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 38028.322, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 65361.438, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.hypothetical-change-is-not-life-event", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 42082.985, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 42722.603, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.deferred-commitment-becomes-ready", "family": "persistent-memory", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 38025.912, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 105339.474, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "loss", "episode_id": "paired.persistent-memory.standing-preference-with-urgent-exception", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 57217.004, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 60839.071, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "loss", "episode_id": "paired.persistent-memory.cross-language-recurring-preference", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 39725.94, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 47818.111, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.event-history-updates-current-counts", "family": "persistent-memory", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 74696.743, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 55321.845, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.selective-retraction-preserves-unrelated-memory", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 41854.153, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 54656.351, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.commitment-conflict-across-planning-threads", "family": "crosssession-authority", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 59605.078, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 77689.265, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.stale-revision-does-not-overwrite-new-work", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 53879.379, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 66229.694, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.response-cancels-only-matching-followup", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 86520.422, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 68669.332, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.crosssession-authority.confirmed-alias-versus-shared-display-name", "family": "crosssession-authority", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 29326.713, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 38619.774, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.claimed-administrator-is-not-a-grant", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 23246.667, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21569.015, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.handoff-plan-is-not-worker-completion", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 42965.477, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 38971.796, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.new-shared-limit-invalidates-old-plan", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 83400.596, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 63228.229, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.nested-config-change-preserves-unrelated-settings", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16697.627, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 28361.685, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.capability-routing-with-unavailable-endpoint", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 22796.389, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 27285.792, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.coding-ops.embedding-revision-invalidates-derived-index", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19954.395, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 23770.327, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "loss", "episode_id": "paired.coding-ops.retention-plan-respects-active-references", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17341.645, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 53410.353, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.base-path-migration-preserves-external-urls", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19152.997, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 56660.039, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.retry-budget-includes-backoff-time", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 25938.602, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 34032.493, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.logging-config-removes-sensitive-fields", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 28201.505, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 38490.351, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.direct-denial-versus-reported-speech", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16588.165, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 19849.126, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.duration-crosses-calendar-midnight", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15952.555, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 32479.035, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.extraction-review.policy-negation-with-narrow-exception", "family": "extraction-review", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 60721.343, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 19917.724, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.uncertain-range-does-not-imply-exact-value", "family": "extraction-review", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 49106.457, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 25844.371, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.extraction-review.forecast-probability-is-not-observed-outcome", "family": "extraction-review", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 15414.818, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 32764.056, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.supported-contradicted-and-unknown-claims", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15758.085, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 19781.145, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.superseded-source-does-not-authorize-action", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 18739.776, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25731.284, "outcome": "pass", "primary_outcome": "pass" } } } ], "evidence_mode": "actual_inference", "kind": "paired_agent_benchmark", "limitations": [ "Public development data, not a held-out benchmark or a comprehensive production qualification.", "Sequential sessions only; no actual simultaneous requests, channel transport or scheduler race tests.", "Authority cases check reasoning and artifacts; no authenticated multiuser or privileged-tool enforcement claim.", "Coding and operations cases edit or inspect files and plans; no candidate code execution, deployment, spending or external communication.", "Memory facts enter through ordinary turns. Forgetting checks later output, not physical erasure from every store.", "Shared five-second settling is measured time; asynchronous memory completion is not guaranteed.", "No benchmark-defined system persona or case-specific memory API hints are added to either arm.", "Endpoint usage is operator-declared; competing traffic may slow the live agent and distort these measurements.", "Public development results do not establish model tiers or a causal improvement estimate.", "Request throughput includes the whole HTTP request, not isolated decode; completion tokens may include reasoning.", "Auxiliary model work is partially observed. Missing observations are not zero.", "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun." ], "manifest_sha256": "5a5f6c5630edfe67aada8ca6d506550b282e897f2404a7491124c4cac142c1fa", "paired_score": { "base_hermes_completed": 49, "base_hermes_completion_percent": 81.66666666666667, "both_completed": 40, "delta_percentage_points": -3.3333333333333335, "episodes": 60, "losses": 9, "neither_completed": 4, "protagine_completed": 47, "protagine_completion_percent": 78.33333333333333, "ties": 44, "wins": 7 }, "published_at": "2026-09-21T18:12:59.796889+00:00", "quality_status": "development", "report_protocol": "paired-attribution-3", "run_id": "paired-5a5f6c5630edfe67aada8ca6", "runtime": { "hermes_payload_sha256": "080ba118c2500e65e6b5c4f2cfe7fa72cd3ce37e6508e5489b4964efc1159bfa", "hermes_version": "0.21.3", "image_id": "sha256:ea1c805d9b8cff55695ac250fa0d4ef54cfca6754c6e8da31ea4314467f4e6d0", "protagine_payload_sha256": "f0e1e455b29fc591c3f08529f58a5d9b9c1c7ecd6df48a0b77752182bdd42b46" }, "schema_version": 1, "unavailable_pairs": 0 }