{ "arms": { "base_hermes": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 60, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 1508620, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 25901, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 60, "observed_total": 1661362.487, "total": 1661362.487 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 283, "total": null } }, "attributed_completed": 55, "completion_percent": 91.66666666666667, "observed_completed": 55, "outcomes": { "fail": 5, "pass": 55 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 60, "max": 86325.965, "median": 21773.627, "min": 14566.83, "samples": 60, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 283, "max": 14117.432, "median": 657.9639999999999, "min": 339.074, "samples": 108, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 283, "max": 1905.211, "median": 373.5955, "min": 238.069, "samples": 282, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 283, "max": 16355.978, "median": 2848.689, "min": 582.183, "samples": 283, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 283, "max": 35.86251997489624, "median": 26.248990541750455, "min": 13.280304865208066, "samples": 283, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 283, "instrumented": 283, "observed": 283, "streaming": 283, "with_first_content": 108, "with_first_generated": 282, "with_usage": 283 } } }, "protagine": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 60, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 2258184, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 44765, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 60, "observed_total": 2477611.228, "total": 2477611.228 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 60, "observed_episodes": 60, "observed_total": 465, "total": null } }, "attributed_completed": 54, "completion_percent": 90.0, "observed_completed": 54, "outcomes": { "fail": 6, "pass": 54 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 60, "max": 119318.31, "median": 31067.5905, "min": 15988.249, "samples": 60, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 291, "max": 63955.79, "median": 2387.6785, "min": 676.458, "samples": 126, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 291, "max": 4504.187, "median": 1008.48, "min": 312.42, "samples": 290, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 395, "max": 97076.534, "median": 3947.353, "min": 901.088, "samples": 395, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 395, "max": 32.59378533920297, "median": 17.905109119588566, "min": 4.724663911032688, "samples": 395, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 395, "instrumented": 465, "observed": 465, "streaming": 291, "with_first_content": 126, "with_first_generated": 290, "with_usage": 395 } } } }, "budget_mode": "deployment_policy", "budget_verification": "unverified", "comparable_pairs": 60, "comparison_key": "f4abf2032d990c047d40b8b7a3d9fc0005ba5bb2e0cd9bc5b116fd223206487a", "completion_projection": { "artifact_verifier_reexecuted": false, "basis": "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun.", "corrected_episodes": 1, "raw_results_preserved": true, "rule": "returned_native_no_output_noncompletion", "source_protocol": "paired-attribution-2" }, "dataset": { "sha256": "9d513d5288be1ef6043a930c66ff1513e810c7fcefb7a58576668e5c804e7127", "source_sha256": "888e6982b9239dd76b35a2f26feb851172ed070c9f932eab1bef431953e09a74", "split": "development", "version": "paired-agent-reviewed-2" }, "declared_episodes": 60, "deployment": { "concurrency": 16, "context_limit": 1048576, "hardware": "NVIDIA DGX Spark GB10, 128 GB per node", "id": "glm53-uncensored-nvfp4-tp16-high", "model": "GLM-5.3 Uncensored", "node_count": 16, "profile": "Frozen text-agent baseline; 4096 output tokens per call; 8 iterations; isolated state per attempt", "quantization": "NVFP4 weights; FP8 MLA KV cache", "revision": "409ddf484d16cb7b13d390e8b30942890164d5af", "runtime": "vLLM with upstream structured-output repair", "runtime_version": "v19 + upstream PR44993", "source_url": "https://huggingface.co/dealignai/GLM-5.3-UNCENSORED-NVFP4", "variant": "uncensored", "weights_verified": true }, "endpoint_usage": "unknown", "episodes": [ { "comparison": "tie", "episode_id": "paired.grounded-evidence.dispatch-is-not-delivery", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 14786.437, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 19560.217, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.inventory-event-deduplication", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 18713.021, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 41327.195, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.unequal-measurement-denominators", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15494.297, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21823.828, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.shared-resource-schedule", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19828.292, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25980.073, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.stale-handoff-reference", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 25001.124, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 25706.779, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.approval-expiry-at-planning-time", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 21958.824, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 33696.217, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.persistent-memory.scoped-correction-in-new-session", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 36680.162, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 47886.234, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.separate-peoples-preferences", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 33724.116, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 51833.55, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.revoked-transient-detail", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 57770.572, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 92401.586, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.sequential-task-handoff", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 45644.851, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 56562.214, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.public-artifact-disclosure-scope", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 37169.374, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 58304.474, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.instructions-inside-untrusted-report", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19258.138, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21657.529, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.python-syntax-repair", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 35779.19, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21704.227, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.bounded-retry-configuration", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16253.89, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 19658.077, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.environment-to-typed-config", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16919.91, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 19519.974, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.quotation-is-not-personal-history", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 18860.903, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 31124.55, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.condition-is-not-authorization", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16445.614, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21626.176, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.review-claims-against-measurements", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 18266.699, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21411.687, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.observation-time-beats-file-order", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 14649.328, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21488.123, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.grounded-evidence.equally-supported-conflict", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17801.043, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 34987.371, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.missing-observations-are-not-zero", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15425.493, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 20514.381, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.grounded-evidence.settled-refunds-and-pending-reversals", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 14566.83, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 15988.249, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.correlation-with-confounded-change", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 21588.43, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 60451.329, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.grounded-evidence.binary-capacity-and-decimal-sizes", "family": "grounded-evidence", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 21197.956, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 26378.248, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.grounded-evidence.half-open-observation-window", "family": "grounded-evidence", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 20822.196, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 31592.755, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.independent-resources-critical-path", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 24466.115, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 23250.524, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.idempotent-resume-plan", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 21372.778, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 22128.643, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.bounded-selection-with-blocked-work", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 23701.523, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 31010.631, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.timeout-reconciliation-before-retry", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 24293.142, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 35456.844, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.file-restoration-after-failed-check", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 38303.414, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 40788.753, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.consent-does-not-expand-action-scope", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 24197.673, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 31584.72, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.planning-toolrecovery.dependency-cycle-with-independent-work", "family": "planning-toolrecovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 25342.036, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 22309.421, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.one-day-exception-does-not-change-default", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 45751.677, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 73362.793, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.hypothetical-change-is-not-life-event", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 37289.264, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 48255.769, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.deferred-commitment-becomes-ready", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 86325.965, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 112710.162, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.standing-preference-with-urgent-exception", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 46533.948, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 53343.772, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.cross-language-recurring-preference", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 41729.723, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 46403.242, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.persistent-memory.event-history-updates-current-counts", "family": "persistent-memory", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 45055.613, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 82649.69, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.persistent-memory.selective-retraction-preserves-unrelated-memory", "family": "persistent-memory", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 37976.083, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 51885.279, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.commitment-conflict-across-planning-threads", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 40559.285, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 119318.31, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.stale-revision-does-not-overwrite-new-work", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 40305.402, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 67950.4, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.crosssession-authority.response-cancels-only-matching-followup", "family": "crosssession-authority", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 43980.756, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 60522.688, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.confirmed-alias-versus-shared-display-name", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 36669.624, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 97615.646, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.claimed-administrator-is-not-a-grant", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17039.831, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 36348.626, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.handoff-plan-is-not-worker-completion", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 45151.857, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 52436.66, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.crosssession-authority.new-shared-limit-invalidates-old-plan", "family": "crosssession-authority", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 49063.04, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 97147.591, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.nested-config-change-preserves-unrelated-settings", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 14738.045, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 20789.657, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.capability-routing-with-unavailable-endpoint", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17700.566, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21396.712, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.embedding-revision-invalidates-derived-index", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 31658.3, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 36579.466, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.retention-plan-respects-active-references", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 24997.451, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 24424.196, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.coding-ops.base-path-migration-preserves-external-urls", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 14989.22, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 108164.453, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.coding-ops.retry-budget-includes-backoff-time", "family": "coding-ops", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 24847.93, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 26121.995, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.coding-ops.logging-config-removes-sensitive-fields", "family": "coding-ops", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17581.934, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 22003.396, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.extraction-review.direct-denial-versus-reported-speech", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 17379.796, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": { "rule": "returned_native_no_output_noncompletion", "source_row_sha256": "d9e99ffb450805ee764f9cca1b8edaa47958878db9bf05e9677f0b7a94023c6b" }, "elapsed_ms": 28091.734, "outcome": "fail", "primary_outcome": "unverified" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.duration-crosses-calendar-midnight", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16655.121, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 20442.224, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.policy-negation-with-narrow-exception", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 16966.23, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21619.312, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.uncertain-range-does-not-imply-exact-value", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 15433.939, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 22425.338, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.forecast-probability-is-not-observed-outcome", "family": "extraction-review", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 19687.793, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 28214.674, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.supported-contradicted-and-unknown-claims", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19373.305, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 21510.735, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.extraction-review.superseded-source-does-not-authorize-action", "family": "extraction-review", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 19637.418, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 26162.129, "outcome": "pass", "primary_outcome": "pass" } } } ], "evidence_mode": "actual_inference", "kind": "paired_agent_benchmark", "limitations": [ "Public development data, not a held-out benchmark or a comprehensive production qualification.", "Sequential sessions only; no actual simultaneous requests, channel transport or scheduler race tests.", "Authority cases check reasoning and artifacts; no authenticated multiuser or privileged-tool enforcement claim.", "Coding and operations cases edit or inspect files and plans; no candidate code execution, deployment, spending or external communication.", "Memory facts enter through ordinary turns. Forgetting checks later output, not physical erasure from every store.", "Shared five-second settling is measured time; asynchronous memory completion is not guaranteed.", "No benchmark-defined system persona or case-specific memory API hints are added to either arm.", "Endpoint usage is operator-declared; competing traffic may slow the live agent and distort these measurements.", "Public development results do not establish model tiers or a causal improvement estimate.", "Request throughput includes the whole HTTP request, not isolated decode; completion tokens may include reasoning.", "Auxiliary model work is partially observed. Missing observations are not zero.", "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun." ], "manifest_sha256": "6fd8e977fb3ef62c2ab4f734df3200064e11b7a7dac2d7a5a74d18f5dde418a0", "paired_score": { "base_hermes_completed": 55, "base_hermes_completion_percent": 91.66666666666667, "both_completed": 50, "delta_percentage_points": -1.6666666666666667, "episodes": 60, "losses": 5, "neither_completed": 1, "protagine_completed": 54, "protagine_completion_percent": 90.0, "ties": 51, "wins": 4 }, "published_at": "2026-09-21T22:42:55.847249+00:00", "quality_status": "development", "report_protocol": "paired-attribution-3", "run_id": "paired-6fd8e977fb3ef62c2ab4f734", "runtime": { "hermes_payload_sha256": "080ba118c2500e65e6b5c4f2cfe7fa72cd3ce37e6508e5489b4964efc1159bfa", "hermes_version": "0.21.3", "image_id": "sha256:ea1c805d9b8cff55695ac250fa0d4ef54cfca6754c6e8da31ea4314467f4e6d0", "protagine_payload_sha256": "f0e1e455b29fc591c3f08529f58a5d9b9c1c7ecd6df48a0b77752182bdd42b46" }, "schema_version": 1, "unavailable_pairs": 0 }