{ "schema_version": 1, "deployment_id": "glm53-flash-abliterated-rtx-tp2-high", "title": "GLM-5.3 Flash modified, two RTX PRO6000 GPUs", "model": { "repository": "lovesenko/GLM-5.3-Flash-tr3-4bpw-Abliterated", "revision": "c8f58e6aa9117c73607d692978b22f091d80450c", "url": "https://huggingface.co/lovesenko/GLM-5.3-Flash-tr3-4bpw-Abliterated", "revision_url": "https://huggingface.co/lovesenko/GLM-5.3-Flash-tr3-4bpw-Abliterated/tree/c8f58e6aa9117c73607d692978b22f091d80450c", "license_declaration": "MIT", "verification": "Mounted STAGE-PLAN repository and NAS-META deployment receipt; not a fresh weight rehash; Repository SHA256SUMS and MANIFEST are stale for 43 abliterated shards according to deployment NAS-META", "download_argv": [ "hf", "download", "lovesenko/GLM-5.3-Flash-tr3-4bpw-Abliterated", "--revision", "c8f58e6aa9117c73607d692978b22f091d80450c", "--local-dir", "{MODEL_DIR}" ] }, "hardware": { "node_count": 1, "gpu_count": 2, "accelerator": "NVIDIA RTX PRO6000 Blackwell Max-Q Workstation", "memory_per_gpu_gib": 96, "memory_type": "GDDR7", "interconnect": "Same-host PCIe", "note": "This describes tested hardware, not a minimum requirement or a throughput guarantee." }, "public_download_path": "/benchmarks/recipes/glm53-flash-abliterated-rtx-tp2-high.json", "record_scope": "Observed serving configuration; independent reproduction not yet performed", "reproduction": { "status": "pinned-public-image-and-recorded-arguments", "limitations": [ "Checkpoint manifest is stale for 43 modified shards; full tensor identity is not established by the repository revision alone.", "Production incumbent was concurrently available to ordinary agent traffic, so screening latency is not an isolated throughput benchmark.", "Public runtime image is pinned and reachable; independent launch reproduction has not been performed.", "The linked upstream launch profile differs from this benchmark. Use the recorded arguments below to reproduce the tested configuration." ] }, "sources": [ "https://huggingface.co/lovesenko/GLM-5.3-Flash-tr3-4bpw-Abliterated/tree/c8f58e6aa9117c73607d692978b22f091d80450c", "https://github.com/brandonmmusic-max/glm-5.3-flash-exl3-4bpw/tree/24784d718cb7bbb99feaee2050eaddcfbc913386", "https://github.com/brandonmmusic-max/glm-5.3-flash-exl3-4bpw/blob/24784d718cb7bbb99feaee2050eaddcfbc913386/README.md", "https://hub.docker.com/r/verdictai/glm53-flash-exl3-k4" ], "placeholders": { "MODEL_DIR": "Checkpoint snapshot directory on each rank or a read-only shared filesystem", "DRAFT_DIR": "Separate pinned draft checkpoint directory when required", "CACHE_DIR": "Writable runtime cache", "NODE_RANK": "Zero-based rank", "NODE_IP": "This node fabric address", "HEAD_IP": "Rank-zero fabric address", "BIND_IP": "Desired API bind address", "API_PORT": "API listen port", "DIST_PORT": "Unused distributed coordination port", "FABRIC_INTERFACE": "Your fabric interface", "RDMA_DEVICES": "Your active RoCE device list", "SERVED_MODEL_NAME": "Public API model name chosen by the deployer" }, "limitations": [ "Configured context and scheduler slots are capacity limits, not proof of simultaneous useful context or latency.", "Use identical pinned weights/runtime across ranks and validate driver, CUDA architecture and RDMA interface compatibility.", "Effort labels have model-specific meanings; high and medium are not cross-model compute units." ], "runtime": { "engine": "vLLM", "version": "0.26.1rc0+infernal.invocation.cu133.r19.vllm174c789.b12x12c4263", "engine_source_tree": "174c789e09984049d0d53b261024460ca5e9c449", "image_id": "sha256:0f1cdcc8891f1cc3a444121eb61d366289a1cbba285f0892dcbb24bc94961692", "pull_reference": "verdictai/glm53-flash-exl3-k4@sha256:0f1cdcc8891f1cc3a444121eb61d366289a1cbba285f0892dcbb24bc94961692", "observed_local_tag": "verdictai/glm53-flash-exl3-k4:r19-sm120-tp2-ep2-dcp2-v84-dflash2", "pull_reference_evidence": "Anonymous registry lookup verified the recorded OCI index, amd64 manifest and image configuration hashes; version and source-tree labels match the observed runtime.", "source_url": "https://github.com/brandonmmusic-max/glm-5.3-flash-exl3-4bpw/tree/24784d718cb7bbb99feaee2050eaddcfbc913386", "build_instructions_url": "https://github.com/brandonmmusic-max/glm-5.3-flash-exl3-4bpw/blob/24784d718cb7bbb99feaee2050eaddcfbc913386/README.md", "oci_index_digest": "sha256:0f1cdcc8891f1cc3a444121eb61d366289a1cbba285f0892dcbb24bc94961692", "amd64_manifest_digest": "sha256:184cfdb86fb08902898999ce5d7101f5711e3138f82b4738ba823145c17f8140", "image_config_digest": "sha256:f28ba4b2192d8306f2ab93be9ea868459f76e2fd5893d4eef9f7cc48f9180578", "source_revision": "24784d718cb7bbb99feaee2050eaddcfbc913386", "pull_argv": [ "docker", "pull", "verdictai/glm53-flash-exl3-k4@sha256:0f1cdcc8891f1cc3a444121eb61d366289a1cbba285f0892dcbb24bc94961692" ], "license_declaration": "LicenseRef-ShapleyMCG-1.0", "image_id_note": "The observed Docker runtime ID is the OCI index digest. The registry image-configuration digest is recorded separately." }, "serving": { "executable": "vllm", "argv": [ "serve", "/model", "--host", "{BIND_IP}", "--port", "{API_PORT}", "--tensor-parallel-size", "2", "--enable-expert-parallel", "--decode-context-parallel-size", "2", "--dcp-comm-backend", "a2a", "--dtype", "bfloat16", "--load-format", "safetensors", "--moe-backend", "b12x", "--attention-backend", "B12X_MLA_SPARSE", "--kv-cache-dtype", "nvfp4_ds_mla", "--gpu-memory-utilization", "0.986", "--enable-chunked-prefill", "--no-enable-prefix-caching", "--generation-config", "/model", "--chat-template", "/opt/glm53/chat_template.multimodal.jinja", "--reasoning-parser", "glm45", "--tool-call-parser", "glm47", "--enable-auto-tool-choice", "--disable-custom-all-reduce", "--speculative-config", "{\"method\":\"dflash\",\"model\":\"/draft\",\"num_speculative_tokens\":3,\"draft_tensor_parallel_size\":2,\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"attention_backend\":\"TRITON_ATTN\",\"kv_cache_dtype\":\"fp8\"}", "--served-model-name", "{SERVED_MODEL_NAME}", "--max-num-batched-tokens", "1024", "--max-num-seqs", "4", "--max-model-len", "327680" ], "argv_note": "Repeated scalar flags normalized to their effective final occurrence", "tp": 2, "ep": true, "dcp": 2, "context_tokens": 327680, "scheduler_slots": 4, "weight_format": "EXL3/TR3 4bpw", "kv_format": "NVFP4 MLA", "speculation": { "method": "DFlash", "draft_tokens": 3, "draft_tp": 2, "draft_kv_format": "FP8" }, "reasoning_parser": "glm45", "tool_parser": "glm47", "request_body": { "chat_template_kwargs": { "reasoning_effort": "high" } }, "container_mounts": [ "{MODEL_DIR}:/model:ro", "{DRAFT_DIR}:/draft:ro", "{CACHE_DIR}:/cache" ], "draft_model": { "repository": "incoai/GLM-5.3-Flash-DFlash2", "revision": "7d74cdd881ed7e32c31175984a67823127b66cfe", "url": "https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2", "revision_url": "https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2/tree/7d74cdd881ed7e32c31175984a67823127b66cfe", "license_declaration": "Not independently checked in this campaign", "verification": "Runtime deployment label; no fresh weight rehash", "download_argv": [ "hf", "download", "incoai/GLM-5.3-Flash-DFlash2", "--revision", "7d74cdd881ed7e32c31175984a67823127b66cfe", "--local-dir", "{MODEL_DIR}" ] } } }