{ "arms": { "base_hermes": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 36, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 36, "observed_total": 5952726, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 36, "observed_total": 85941, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 36, "observed_total": 4586331.109, "total": 4586331.109 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 36, "observed_total": 746, "total": null } }, "attributed_completed": 0, "completion_percent": 0.0, "observed_completed": 0, "outcomes": { "fail": 36 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 36, "max": 267741.479, "median": 115832.10500000001, "min": 41452.681, "samples": 36, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 718, "max": 18827.601, "median": 3080.518, "min": 2063.078, "samples": 160, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 718, "max": 22161.95, "median": 2589.8954999999996, "min": 1613.094, "samples": 714, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 746, "max": 27942.231, "median": 3708.3514999999998, "min": 1931.983, "samples": 746, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 746, "max": 98.55718763187664, "median": 19.600948395608548, "min": 0.6745233952593777, "samples": 746, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 746, "instrumented": 746, "observed": 746, "streaming": 718, "with_first_content": 160, "with_first_generated": 714, "with_usage": 746 } } }, "protagine": { "accounting": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 36, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 36, "observed_total": 9641461, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 36, "observed_total": 218892, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 36, "observed_total": 7048932.89, "total": 7048932.89 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 36, "observed_episodes": 36, "observed_total": 1283, "total": null } }, "attributed_completed": 8, "completion_percent": 22.22222222222222, "observed_completed": 8, "outcomes": { "fail": 28, "pass": 8 }, "timing": { "coverage": "Observed HTTP model requests across foreground and background work; coverage is incomplete and endpoint traffic may be shared. Request timing is not user-turn latency or serving capacity.", "decode_tokens_per_second": null, "metrics": { "episode_elapsed_ms": { "definition": "Runner wall time per arm episode, including model calls, tools, fixed settling, container startup and cleanup. Observed failed attempts are included.", "eligible_samples": 36, "max": 380445.662, "median": 226133.653, "min": 45311.568, "samples": 36, "unit": "ms" }, "first_content_ms": { "definition": "Client-observed first nonempty nonreasoning text in completed SSE responses. Tool-only responses have no value; this is not whole-task final-answer latency.", "eligible_samples": 784, "max": 22324.564, "median": 4481.368, "min": 2860.94, "samples": 206, "unit": "ms" }, "first_generated_ms": { "definition": "Client-observed first nonempty reasoning, text or tool payload in completed SSE responses. Role and empty frames are excluded; this is not server-internal token timing.", "eligible_samples": 784, "max": 24385.77, "median": 3774.2335000000003, "min": 1975.293, "samples": 776, "unit": "ms" }, "request_elapsed_ms": { "definition": "Client-observed request wall time through response completion/close for completed HTTP 2xx responses, including queueing, prefill, generation and client response consumption.", "eligible_samples": 1190, "max": 56781.148, "median": 5536.9195, "min": 990.942, "samples": 1190, "unit": "ms" }, "request_output_tokens_per_second": { "definition": "Provider-reported completion tokens divided by client-observed request seconds, for completed HTTP 2xx responses with token usage. Counts may include reasoning and tool tokens; this is end-to-end output TPS, not decode TPS or aggregate serving throughput.", "eligible_samples": 1190, "max": 85.47473004702807, "median": 19.879194163880904, "min": 0.7881869689836605, "samples": 1190, "unit": "tokens/s" } }, "protocol": "paired-transport-2", "requests": { "completed": 1190, "instrumented": 1283, "observed": 1283, "streaming": 784, "with_first_content": 206, "with_first_generated": 776, "with_usage": 1190 } } } }, "budget_mode": "deployment_policy", "budget_verification": "unverified", "comparable_pairs": 36, "comparison_key": "62635aa43d8bdd432d221583b8ecde9881a08d509049c661e9596956c3d68d2c", "completion_projection": { "artifact_verifier_reexecuted": false, "basis": "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun.", "corrected_episodes": 5, "raw_results_preserved": true, "rule": "returned_native_no_output_noncompletion", "source_protocol": "paired-attribution-2" }, "dataset": { "sha256": "1f6bff097336bb13ead67c9caa93cf0d2cad8e3e5098160b7680c05984d84ab6", "source_sha256": "93f39753e75ea945da5199fb7a006497d76db28fef7d678682877c956f997c1d", "split": "frozen_public_evaluation", "version": "paired-agent-workflows-1" }, "declared_episodes": 36, "deployment": { "concurrency": 4, "context_limit": 327680, "hardware": "Two NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation GPUs, 96 GB per GPU", "id": "glm53-flash-abliterated-rtx-tp2-high", "model": "GLM-5.3 Flash Abliterated", "node_count": 1, "profile": "Frozen text-agent baseline; 4096 output tokens per call; 8 iterations; isolated state per attempt", "quantization": "EXL3 4 bpw weights; NVFP4 MLA KV cache", "revision": "c8f58e6aa9117c73607d692978b22f091d80450c", "runtime": "vLLM", "runtime_version": "0.26.1rc0+infernal.invocation.cu133.r19.vllm174c789.b12x12c4263", "source_url": "https://huggingface.co/lovesenko/GLM-5.3-Flash-tr3-4bpw-Abliterated", "variant": "abliterated", "weights_verified": false }, "endpoint_usage": "shared", "episodes": [ { "comparison": "tie", "episode_id": "paired.workflow-recall.restock-handoff.repeat-01", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 68803.09, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 176926.739, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.current-restock-record.repeat-01", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 205994.955, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 231748.96, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.proposal-does-not-replace-commitment.repeat-01", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 55095.213, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 59213.653, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.selective-delivery-correction.repeat-01", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 127463.97, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": { "rule": "returned_native_no_output_noncompletion", "source_row_sha256": "4a5665bc7c1eed020ac7f0db4a48c3532166833bccd4541683bd0ad4dc67e06f" }, "elapsed_ms": 78635.34, "outcome": "fail", "primary_outcome": "unverified" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.venue-conflict-before-clarification.repeat-01", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 48189.037, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 358604.652, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-correction.current-approved-schedule.repeat-01", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 177150.17, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 204467.548, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.resume-stable-task-after-read-fault.repeat-01", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 241202.5, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": { "rule": "returned_native_no_output_noncompletion", "source_row_sha256": "56a586df79947e2d09a73c880c271dc879e84231248454d633ee8debd38be35f" }, "elapsed_ms": 45311.568, "outcome": "fail", "primary_outcome": "unverified" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.due-followup-without-duplicate-send.repeat-01", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 136968.717, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 61639.254, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.current-board-after-read-fault.repeat-01", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 264103.923, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 341529.326, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.same-name-owner-linked-contacts.repeat-01", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 45518.85, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 66825.245, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.public-brief-after-private-work.repeat-01", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 122858.717, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 231252.226, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.current-directory-scoped-handoff.repeat-01", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 49390.752, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 73030.075, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-recall.restock-handoff.repeat-02", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 102154.974, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 282134.67, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.workflow-recall.current-restock-record.repeat-02", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 126113.515, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 300450.123, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.proposal-does-not-replace-commitment.repeat-02", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 254039.297, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 222224.885, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.selective-delivery-correction.repeat-02", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 189531.457, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 180977.065, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-correction.venue-conflict-before-clarification.repeat-02", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 267741.479, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 380445.662, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.current-approved-schedule.repeat-02", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 247673.487, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 248071.068, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.resume-stable-task-after-read-fault.repeat-02", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 69331.508, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 73347.189, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.due-followup-without-duplicate-send.repeat-02", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 50277.1, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 302378.36, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.current-board-after-read-fault.repeat-02", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 124567.279, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 291910.745, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.same-name-owner-linked-contacts.repeat-02", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 44439.579, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": { "rule": "returned_native_no_output_noncompletion", "source_row_sha256": "b955e491a75ce19ee7a7fabd87728ce679be5739fa4b0188dc492c3f218954c2" }, "elapsed_ms": 46294.548, "outcome": "fail", "primary_outcome": "unverified" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.public-brief-after-private-work.repeat-02", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 104276.515, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 61144.223, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.current-directory-scoped-handoff.repeat-02", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 44318.618, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 286776.772, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.restock-handoff.repeat-03", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": { "rule": "returned_native_no_output_noncompletion", "source_row_sha256": "b2cd918159d6739954cdbcc53adab5be4dab203cd81b8b430a5d3b8afac82a74" }, "elapsed_ms": 163762.007, "outcome": "fail", "primary_outcome": "unverified" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 223487.299, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.current-restock-record.repeat-03", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 87892.521, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 285485.014, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recall.proposal-does-not-replace-commitment.repeat-03", "family": "workflow-recall", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 198139.531, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 121273.241, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-correction.selective-delivery-correction.repeat-03", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 123751.848, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 283540.489, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "win", "episode_id": "paired.workflow-correction.venue-conflict-before-clarification.repeat-03", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 222927.209, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 319900.598, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-correction.current-approved-schedule.repeat-03", "family": "workflow-correction", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 46845.998, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 272754.502, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-recovery.resume-stable-task-after-read-fault.repeat-03", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": { "rule": "returned_native_no_output_noncompletion", "source_row_sha256": "231d2fbbd14cb1f02c3ef518032d8ec72f65229cf5783f27ee8346d199487333" }, "elapsed_ms": 95785.244, "outcome": "fail", "primary_outcome": "unverified" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 255866.423, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.due-followup-without-duplicate-send.repeat-03", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 195413.799, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 50112.738, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-recovery.current-board-after-read-fault.repeat-03", "family": "workflow-recovery", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 108805.493, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 288082.113, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.same-name-owner-linked-contacts.repeat-03", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 52466.932, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 65565.059, "outcome": "fail", "primary_outcome": "fail" } } }, { "comparison": "win", "episode_id": "paired.workflow-scope.public-brief-after-private-work.repeat-03", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 81883.144, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": true, "completion_projection": null, "elapsed_ms": 228780.007, "outcome": "pass", "primary_outcome": "pass" } } }, { "comparison": "tie", "episode_id": "paired.workflow-scope.current-directory-scoped-handoff.repeat-03", "family": "workflow-scope", "results": { "base_hermes": { "completion": false, "completion_projection": null, "elapsed_ms": 41452.681, "outcome": "fail", "primary_outcome": "fail" }, "protagine": { "completion": false, "completion_projection": null, "elapsed_ms": 48745.511, "outcome": "fail", "primary_outcome": "fail" } } } ], "evidence_mode": "actual_inference", "kind": "paired_agent_benchmark", "limitations": [ "Public frozen synthetic workflow, not a secret holdout or unseen-task claim.", "Sessions are sequential. Process restart is real; no host failure, simultaneous-session race or live channel transport is exercised.", "Only declared local artifacts and checkpoints are graded; no external action or scheduled outreach is executed.", "Memory-relevant tasks permit legitimate workspace notes and native session retrieval in both arms; no mechanism is required or rewarded.", "Exactly twelve small workflows do not establish production qualification, real-time autonomy, multimodal competence, statistical superiority or long-term learning.", "Synthetic read fault targets native read_file; alternative legitimate access can bypass it and must be reported rather than credited as fault recovery.", "Foreground output and private stores are outside these artifact checks; no all-channel confidentiality or physical-forgetting claim.", "Source checks detect changed bytes at declared checkpoints and final capture, not a modification restored between snapshots.", "Checkpoints and read faults are part of this versioned protocol. Do not silently edit frozen inputs or graders after inspecting model results.", "Endpoint usage is operator-declared; competing traffic may slow the live agent and distort these measurements.", "Public development results do not establish model tiers or a causal improvement estimate.", "Request throughput includes the whole HTTP request, not isolated decode; completion tokens may include reasoning.", "Auxiliary model work is partially observed. Missing observations are not zero.", "Report-only attribution correction, applied symmetrically to both arms. A returned native container with empty output counts as noncompletion only with independent serialized candidate requests and returned-model evidence, no fallback, confirmed cleanup, and consistent recorded native incomplete turns. Saved outcomes, primary attribution and checks are unchanged; artifact grading is not rerun." ], "manifest_sha256": "75eae932f025b666768c2eea6319d821f7077ab11665173a941a4d4ac82ce786", "paired_score": { "base_hermes_completed": 0, "base_hermes_completion_percent": 0.0, "both_completed": 0, "delta_percentage_points": 22.22222222222222, "episodes": 36, "losses": 0, "neither_completed": 28, "protagine_completed": 8, "protagine_completion_percent": 22.22222222222222, "ties": 28, "wins": 8 }, "published_at": "2026-09-21T18:13:00.334058+00:00", "quality_status": "frozen_public_evaluation", "report_protocol": "paired-attribution-3", "run_id": "paired-75eae932f025b666768c2eea", "runtime": { "hermes_payload_sha256": "080ba118c2500e65e6b5c4f2cfe7fa72cd3ce37e6508e5489b4964efc1159bfa", "hermes_version": "0.21.3", "image_id": "sha256:ea1c805d9b8cff55695ac250fa0d4ef54cfca6754c6e8da31ea4314467f4e6d0", "protagine_payload_sha256": "f0e1e455b29fc591c3f08529f58a5d9b9c1c7ecd6df48a0b77752182bdd42b46" }, "schema_version": 1, "unavailable_pairs": 0, "workflow_repetitions": { "basis": "Descriptive repeat counts grouped by frozen workflow. Repetitions are not independent scenario designs. No confidence interval or generalization claim; public cases are not a private holdout.", "unique_workflows": 12, "workflows": [ { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 398674, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 4043, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 334720.071, "total": 334720.071 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 57, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 788993, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 22402, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 682548.708, "total": 682548.708 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 124, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "format": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "native_completion": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "semantic": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } } }, "family": "workflow-recall", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 1 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-recall.restock-handoff" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 615299, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 7984, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 420000.991, "total": 420000.991 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 71, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 1100215, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 24610, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 817684.0970000001, "total": 817684.0970000001 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 149, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } } }, "family": "workflow-recall", "losses": 0, "memory_condition": "irrelevant", "successes": { "base_hermes": 0, "protagine": 1 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-recall.current-restock-record" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 635105, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 8961, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 507274.04099999997, "total": 507274.04099999997 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 80, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 535542, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 12613, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 402711.779, "total": 402711.779 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 64, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } } }, "family": "workflow-recall", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 0 }, "ties": 3, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 0, "workflow_id": "paired.workflow-recall.proposal-does-not-replace-commitment" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 527015, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 7404, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 440747.275, "total": 440747.275 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 70, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 790704, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 16781, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 543152.894, "total": 543152.894 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 105, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } } }, "family": "workflow-correction", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 1 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-correction.selective-delivery-correction" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 752904, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 10530, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 538857.725, "total": 538857.725 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 82, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 1477178, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 31125, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 1058950.912, "total": 1058950.912 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 174, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 2 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } } }, "family": "workflow-correction", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 2 }, "ties": 1, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 2, "workflow_id": "paired.workflow-correction.venue-conflict-before-clarification" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 612483, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 11298, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 471669.655, "total": 471669.655 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 71, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 901353, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 22944, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 725293.118, "total": 725293.118 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 136, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } } }, "family": "workflow-correction", "losses": 0, "memory_condition": "irrelevant", "successes": { "base_hermes": 0, "protagine": 1 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-correction.current-approved-schedule" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 505413, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 7079, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 406319.252, "total": 406319.252 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 68, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 592755, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 12471, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 374525.18, "total": 374525.18 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 74, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "format": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "native_completion": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } }, "semantic": { "base_hermes": { "observed_repetitions": 2, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 1 } } }, "family": "workflow-recovery", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 1 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-recovery.resume-stable-task-after-read-fault" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 523709, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 8642, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 382659.616, "total": 382659.616 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 64, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 596634, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 12034, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 414130.35199999996, "total": 414130.35199999996 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 76, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } } }, "family": "workflow-recovery", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 0 }, "ties": 3, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 0, "workflow_id": "paired.workflow-recovery.due-followup-without-duplicate-send" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 620478, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 9070, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 497476.695, "total": 497476.695 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 77, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 1286201, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 29138, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 921522.184, "total": 921522.184 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 166, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 3 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 1 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } } }, "family": "workflow-recovery", "losses": 0, "memory_condition": "irrelevant", "successes": { "base_hermes": 0, "protagine": 0 }, "ties": 3, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 0, "workflow_id": "paired.workflow-recovery.current-board-after-read-fault" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 190529, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 3147, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 142425.361, "total": 142425.361 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 27, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 237989, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 4639, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 178684.85199999998, "total": 178684.85199999998 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 30, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 0 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 0 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 0 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 0 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 2, "passed_repetitions": 0 } } }, "family": "workflow-scope", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 0 }, "ties": 3, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 0, "workflow_id": "paired.workflow-scope.same-name-owner-linked-contacts" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 369898, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 5077, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 309018.376, "total": 309018.376 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 52, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 764887, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 18630, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 521176.456, "total": 521176.456 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 110, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 2 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } } }, "family": "workflow-scope", "losses": 0, "memory_condition": "relevant", "successes": { "base_hermes": 0, "protagine": 1 }, "ties": 2, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 1, "workflow_id": "paired.workflow-scope.public-brief-after-private-work" }, { "accounting": { "base_hermes": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 201219, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 2706, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 135162.051, "total": 135162.051 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 27, "total": null } }, "protagine": { "background_model_calls": { "coverage": "unobserved", "declared_episodes": 3, "observed_episodes": 0, "observed_total": null, "total": null }, "input_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 569010, "total": null }, "output_tokens": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 11505, "total": null }, "runner_elapsed_ms": { "basis": "Arm wall time includes tools, settling and container cleanup; not model-only latency.", "observed_episodes": 3, "observed_total": 408552.358, "total": 408552.358 }, "total_model_calls": { "coverage": "partial", "declared_episodes": 3, "observed_episodes": 3, "observed_total": 75, "total": null } } }, "comparable_repetitions": 3, "declared_repetitions": 3, "dimensions": { "checkpoints": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "format": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } }, "lifecycle": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "native_completion": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 0 } }, "semantic": { "base_hermes": { "observed_repetitions": 3, "passed_repetitions": 0 }, "protagine": { "observed_repetitions": 3, "passed_repetitions": 1 } } }, "family": "workflow-scope", "losses": 0, "memory_condition": "irrelevant", "successes": { "base_hermes": 0, "protagine": 0 }, "ties": 3, "unavailable": { "base_hermes": 0, "protagine": 0 }, "wins": 0, "workflow_id": "paired.workflow-scope.current-directory-scoped-handoff" } ] } }