{ "arms": { "base_hermes": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 36, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 35, "observed_total": 6842094, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 35, "observed_total": 244699, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 36, "observed_total": 11705837.072, "total": 11705837.072 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 35, "observed_total": 862, "total": null } }, "attributed_completed": 14, "completion_percent": null, "observed_completed": 14, "outcomes": { "fail": 21, "pass": 14, "timeout": 1 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 36, "max": 600344.059, "median": 323505.36, "min": 133219.531, "samples": 36, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 856, "max": 119725.806, "median": 3440.6465, "min": 424.574, "samples": 426, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 856, "max": 13067.346, "median": 507.958, "min": 280.649, "samples": 853, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 862, "max": 132734.597, "median": 6215.7855, "min": 680.229, "samples": 862, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 862, "max": 37.133973697787994, "median": 26.162991892036505, "min": 6.07182744170002, "samples": 862, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 862, "instrumented": 862, "observed": 862, "streaming": 856, "with_first_content": 426, "with_first_generated": 853, "with_usage": 862 } } }, "protagine": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 36, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 27, "observed_total": 7061127, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 27, "observed_total": 221127, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 36, "observed_total": 16125282.44, "total": 16125282.44 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 27, "observed_total": 1063, "total": null } }, "attributed_completed": 17, "completion_percent": null, "observed_completed": 17, "outcomes": { "fail": 10, "pass": 17, "timeout": 9 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 36, "max": 600656.851, "median": 429736.7405, "min": 152764.403, "samples": 36, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 598, "max": 99079.316, "median": 9089.9085, "min": 1429.161, "samples": 274, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 598, "max": 15520.567, "median": 3884.5245, "min": 309.812, "samples": 598, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 885, "max": 186078.972, "median": 10773.672, "min": 935.86, "samples": 885, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 885, "max": 34.80737283442765, "median": 15.624345566408131, "min": 1.865018926102355, "samples": 885, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 885, "instrumented": 1063, "observed": 1063, "streaming": 598, "with_first_content": 274, "with_first_generated": 598, "with_usage": 885 } } } }, "budget_mode": "deployment_policy", "budget_verification": "unverified", "comparable_pairs": 27, "comparison_key": "fa0f6843e3435beb15923c01c10f48e12b8a714e31fa1b8e31dc5e6fcde59d1e", "completion_projection": { "artifact_verifier_reexecuted": false, "basis": "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun.", "corrected_episodes": 0, "raw_results_preserved": true, "rule": "returned_native_no_output_noncompletion", "source_protocol": "paired-attribution-2" }, "dataset": { "sha256": "1f6bff097336bb13ead67c9caa93cf0d2cad8e3e5098160b7680c05984d84ab6", "source_sha256": "93f39753e75ea945da5199fb7a006497d76db28fef7d678682877c956f997c1d", "split": "frozen_public_evaluation", "version": "paired-agent-workflows-1" }, "declared_episodes": 36, "deployment": { "concurrency": 16, "context_limit": 1048576, "hardware": "NVIDIA DGX Spark GB10, 128 GB per node", "id": "glm53-uncensored-nvfp4-tp16-high", "model": "GLM-5.3 Uncensored", "node_count": 16, "profile": "Frozen text-agent baseline; 4096 output tokens per call; 8 iterations; isolated state per attempt", "quantization": "NVFP4 weights; FP8 MLA KV cache", "revision": "409ddf484d16cb7b13d390e8b30942890164d5af", "runtime": "vLLM with upstream structured-output repair", "runtime_version": "v19 + upstream PR44993", "source_url": "https://huggingface.co/dealignai/GLM-5.3-UNCENSORED-NVFP4", "variant": "uncensored", "weights_verified": true }, "endpoint_usage": "unknown", "episodes": [ { "comparison": "tie", "episode_id": "paired.workflow-recall.restock-handoff.repeat-01", "family": "workflow-recall", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 363515.611, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 270507.647, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "loss", "episode_id": "paired.workflow-recall.current-restock-record.repeat-01", "family": "workflow-recall", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 427400.993, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 152764.403, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "loss", "episode_id": "paired.workflow-recall.proposal-does-not-replace-commitment.repeat-01", "family": "workflow-recall", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 343389.831, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 415113.629, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-correction.selective-delivery-correction.repeat-01", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 374932.323, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 493246.626, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.venue-conflict-before-clarification.repeat-01", "family": "workflow-correction", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 312956.155, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 499166.013, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.workflow-correction.current-approved-schedule.repeat-01", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 301956.464, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 529606.552, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.workflow-recovery.resume-stable-task-after-read-fault.repeat-01", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 300136.416, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 523814.544, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-recovery.due-followup-without-duplicate-send.repeat-01", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 404916.141, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600291.241, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "win", "episode_id": "paired.workflow-recovery.current-board-after-read-fault.repeat-01", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 365920.987, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 304440.102, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-scope.same-name-owner-linked-contacts.repeat-01", "family": "workflow-scope", "results": { "base_hermes": { "completion": null, "completion_projection": null, "elapsed_ms": 600344.059, "outcome": "timeout", "primary_outcome": "unverified" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600421.037, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "loss", "episode_id": "paired.workflow-scope.public-brief-after-private-work.repeat-01", "family": "workflow-scope", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 238975.065, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 355043.016, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "loss", "episode_id": "paired.workflow-scope.current-directory-scoped-handoff.repeat-01", "family": "workflow-scope", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 432571.75, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 444255.358, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.restock-handoff.repeat-02", "family": "workflow-recall", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 288141.566, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 364459.542, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.current-restock-record.repeat-02", "family": "workflow-recall", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 416871.863, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 487939.964, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.proposal-does-not-replace-commitment.repeat-02", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 344172.828, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 249179.455, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.selective-delivery-correction.repeat-02", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 133219.531, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 319654.009, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-correction.venue-conflict-before-clarification.repeat-02", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 145878.961, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600433.457, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.current-approved-schedule.repeat-02", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 303103.572, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 322352.08, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.resume-stable-task-after-read-fault.repeat-02", "family": "workflow-recovery", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 309185.687, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 404351.58, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-recovery.due-followup-without-duplicate-send.repeat-02", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 350288.339, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600513.549, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "win", "episode_id": "paired.workflow-recovery.current-board-after-read-fault.repeat-02", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 320656.307, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 380900.779, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-scope.same-name-owner-linked-contacts.repeat-02", "family": "workflow-scope", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 407361.703, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600565.475, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "win", "episode_id": "paired.workflow-scope.public-brief-after-private-work.repeat-02", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 226018.749, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 415218.123, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.current-directory-scoped-handoff.repeat-02", "family": "workflow-scope", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 257859.434, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 328381.601, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.workflow-recall.restock-handoff.repeat-03", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 184449.574, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 412393.447, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-recall.current-restock-record.repeat-03", "family": "workflow-recall", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 419245.443, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600535.988, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "win", "episode_id": "paired.workflow-recall.proposal-does-not-replace-commitment.repeat-03", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 336468.805, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 476251.479, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-correction.selective-delivery-correction.repeat-03", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 306460.128, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600312.933, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-correction.venue-conflict-before-clarification.repeat-03", "family": "workflow-correction", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 330832.802, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600656.851, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "win", "episode_id": "paired.workflow-correction.current-approved-schedule.repeat-03", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 326354.413, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 399061.114, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.resume-stable-task-after-read-fault.repeat-03", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 266414.357, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 374706.8, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "unavailable", "episode_id": "paired.workflow-recovery.due-followup-without-duplicate-send.repeat-03", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 385096.186, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": null, "completion_projection": null, "elapsed_ms": 600621.69, "outcome": "timeout", "primary_outcome": "unverified" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.current-board-after-read-fault.repeat-03", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 307955.097, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 460408.729, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.same-name-owner-linked-contacts.repeat-03", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 441388.071, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 550673.964, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-scope.public-brief-after-private-work.repeat-03", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 185993.535, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 377090.691, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.current-directory-scoped-handoff.repeat-03", "family": "workflow-scope", "results": { "base_hermes": { "completion": true, "completion_projection": null, "elapsed_ms": 245404.326, "outcome": "pass", "primary_outcome": "pass" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 409948.972, "outcome": "pass", "primary_outcome": "pass" } } } ], "evidence_mode": "actual_inference", "kind": "paired_agent_benchmark", "limitations": [ "Public frozen synthetic workflow, not a secret holdout or unseen-task claim.", "Sessions are sequential. Process restart is real; no host failure, simultaneous-session race or live channel transport is exercised.", "Only declared local artifacts and checkpoints are graded; no external action or scheduled outreach is executed.", "Memory-relevant tasks permit legitimate workspace notes and native session retrieval in both arms; no mechanism is required or rewarded.", "Exactly twelve small workflows do not establish production qualification, real-time autonomy, multimodal competence, statistical superiority or long-term learning.", "Synthetic read fault targets native read_file; alternative legitimate access can bypass it and must be reported rather than credited as fault recovery.", "Foreground output and private stores are outside these artifact checks; no all-channel confidentiality or physical-forgetting claim.", "Source checks detect changed bytes at declared checkpoints and final capture, not a modification restored between snapshots.", "Checkpoints and read faults are part of this versioned protocol. Do not silently edit frozen inputs or graders after inspecting model results.", "Endpoint usage is operator-declared; competing traffic may slow the live agent and distort these measurements.", "Public development results do not establish model tiers or a causal improvement estimate.", "Request throughput includes the whole HTTP request, not isolated decode; completion tokens may include reasoning.", "Auxiliary model work is partially observed. Missing observations are not zero.", "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun." ], "manifest_sha256": "c4c0b566af844fdab48c6d1ad762316f742c7b1cbea2488353b1153da27586c1", "paired_score": null, "published_at": "2026-09-21T22:42:56.475410+00:00", "quality_status": "frozen_public_evaluation", "report_protocol": "paired-attribution-3", "run_id": "paired-c4c0b566af844fdab48c6d1a", "runtime": { "hermes_payload_sha256": "080ba118c2500e65e6b5c4f2cfe7fa72cd3ce37e6508e5489b4964efc1159bfa", "hermes_version": "0.21.3", "image_id": "sha256:ea1c805d9b8cff55695ac250fa0d4ef54cfca6754c6e8da31ea4314467f4e6d0", "protagine_payload_sha256": "f0e1e455b29fc591c3f08529f58a5d9b9c1c7ecd6df48a0b77752182bdd42b46" }, "schema_version": 1, "unavailable_pairs": 9, "workflow_repetitions": { "basis": "Descriptive repeat counts grouped by frozen workflow. Repetitions are not independent scenario designs. No confidence interval or generalization claim; public cases are not a private holdout.", "unique_workflows": 12, "workflows": [ { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 558909, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 17516, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 836106.7509999999, "total": 836106.7509999999 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 69, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 684475, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 19635, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1047360.6359999999, "total": 1047360.6359999999 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 110, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } } }, "family": "workflow-recall", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 2, "protagine": 3 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-recall.restock-handoff" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 548828, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 27851, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1263518.299, "total": 1263518.299 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 72, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 476456, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 15753, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1241240.355, "total": 1241240.355 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 63, "total": null } } }, "comparable_repetitions": 2, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } } }, "family": "workflow-recall", "losses": 1, "memory_condition": "irrelevant", "successes": { "base_hermes": 3, "protagine": 1 }, "ties": 1, "unavailable": { "base_hermes": 0, "protagine": 1 }, "wins": 0, "workflow_id": "paired.workflow-recall.current-restock-record" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 538482, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 22521, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1024031.4639999999, "total": 1024031.4639999999 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 73, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 744162, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 23953, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1140544.563, "total": 1140544.563 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 113, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } } }, "family": "workflow-recall", "losses": 1, "memory_condition": "relevant", "successes": { "base_hermes": 1, "protagine": 1 }, "ties": 1, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-recall.proposal-does-not-replace-commitment" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 549207, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 17021, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 814611.982, "total": 814611.982 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 71, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 528978, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 16358, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1413213.568, "total": 1413213.568 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 77, "total": null } } }, "comparable_repetitions": 2, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } } }, "family": "workflow-correction", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 1 }, "ties": 1, "unavailable": { "base_hermes": 0, "protagine": 1 }, "wins": 1, "workflow_id": "paired.workflow-correction.selective-delivery-correction" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 604422, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 17457, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 789667.9180000001, "total": 789667.9180000001 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 77, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 1, "observed_total": 245707, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 1, "observed_total": 11726, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1700256.321, "total": 1700256.321 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 1, "observed_total": 43, "total": null } } }, "comparable_repetitions": 1, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 1 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 1 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 1 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 1 } } }, "family": "workflow-correction", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 2, "protagine": 1 }, "ties": 1, "unavailable": { "base_hermes": 0, "protagine": 2 }, "wins": 0, "workflow_id": "paired.workflow-correction.venue-conflict-before-clarification" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 545219, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 20342, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 931414.449, "total": 931414.449 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 73, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 916336, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 25326, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1251019.746, "total": 1251019.746 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 133, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } } }, "family": "workflow-correction", "losses": 0, "memory_condition": "irrelevant", "successes": { "base_hermes": 0, "protagine": 2 }, "ties": 1, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 2, "workflow_id": "paired.workflow-correction.current-approved-schedule" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 716139, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 18852, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 875736.46, "total": 875736.46 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 84, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 909758, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 22017, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1302872.924, "total": 1302872.924 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 131, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } } }, "family": "workflow-recovery", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 1, "protagine": 2 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-recovery.resume-stable-task-after-read-fault" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 708273, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 26892, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1140300.666, "total": 1140300.666 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 86, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "output_tokens": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1801426.48, "total": 1801426.48 }, "total_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null } } }, "comparable_repetitions": 0, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 0, "passed_repetitions": 0 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 0, "passed_repetitions": 0 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 0, "passed_repetitions": 0 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 0, "passed_repetitions": 0 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 0, "passed_repetitions": 0 } } }, "family": "workflow-recovery", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 0 }, "ties": 0, "unavailable": { "base_hermes": 0, "protagine": 3 }, "wins": 0, "workflow_id": "paired.workflow-recovery.due-followup-without-duplicate-send" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 570218, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 22383, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 994532.3910000001, "total": 994532.3910000001 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 73, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 825498, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 23519, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1145749.6099999999, "total": 1145749.6099999999 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 122, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } } }, "family": "workflow-recovery", "losses": 0, "memory_condition": "irrelevant", "successes": { "base_hermes": 0, "protagine": 2 }, "ties": 1, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 2, "workflow_id": "paired.workflow-recovery.current-board-after-read-fault" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 515728, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 19161, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1449093.833, "total": 1449093.833 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 2, "observed_total": 55, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 1, "observed_total": 203942, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 1, "observed_total": 12531, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1751660.476, "total": 1751660.476 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 1, "observed_total": 32, "total": null } } }, "comparable_repetitions": 1, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 0 } }, "format": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 0 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 0 } }, "native_completion": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 0 } }, "semantic": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 1, "passed_repetitions": 0 } } }, "family": "workflow-scope", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 1, "protagine": 0 }, "ties": 1, "unavailable": { "base_hermes": 1, "protagine": 2 }, "wins": 0, "workflow_id": "paired.workflow-scope.same-name-owner-linked-contacts" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 429365, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 13465, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 650987.349, "total": 650987.349 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 59, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 716184, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 23423, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1147351.83, "total": 1147351.83 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 117, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } } }, "family": "workflow-scope", "losses": 1, "memory_condition": "relevant", "successes": { "base_hermes": 1, "protagine": 2 }, "ties": 0, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 2, "workflow_id": "paired.workflow-scope.public-brief-after-private-work" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 557304, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 21238, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 935835.51, "total": 935835.51 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 70, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 809631, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 26886, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1182585.931, "total": 1182585.931 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 122, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 3 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } } }, "family": "workflow-scope", "losses": 1, "memory_condition": "irrelevant", "successes": { "base_hermes": 3, "protagine": 2 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 0, "workflow_id": "paired.workflow-scope.current-directory-scoped-handoff" } ] } }