{ "experiment": "2026-10-02-coastal-cat-memory-off", "date": "2026-10-02", "prompt": "全新创建一个可直接运行的 Three.js 网页 3D 场景：内容是一个猫在海边马路上骑电动车的3D动画", "subjects": [ { "label": "A", "model": "gpt-6.1-sol", "effort": "xhigh", "started_local": "2026-10-02T09:12:39+08:00", "completed_local": "2026-10-02T09:34:25+08:00", "duration_ms": 1305531, "first_token_ms": 7583, "model_requests": 42, "tokens": { "input_tokens": 3981686, "cached_input_tokens": 3879808, "cache_write_input_tokens": 0, "output_tokens": 32782, "reasoning_output_tokens": 8549, "total_tokens": 4014468 }, "standard_usd": 0.9195568, "standard_credits": 22.98892, "cost_parts": { "uncached_input": 0.20375600000000002, "cached_input": 0.38798079999999996, "output": 0.32781999999999994 }, "max_request_input": 118587, "cache_hit": 0.97441335152998, "name": "GPT-6.1 Sol", "display": "A · GPT-6.1 Sol / xhigh", "time_display": "21:46", "quality_rank": 2, "memory": { "database_mode": "disabled", "summary_injected": false, "observed_history_memory_access": false, "observed_memory_skill_read": false, "ordinary_skills": [ "Sites building", "Sites hosting", "Computer use" ] }, "service_tier": "not recorded; normalized Standard cost only", "original_page": "scenes/A.html", "original_source_zip": "sources/A.zip", "source_review": "code-review/A.html" }, { "label": "B", "model": "gpt-6-astra", "effort": "xhigh", "started_local": "2026-10-02T09:18:18+08:00", "completed_local": "2026-10-02T09:33:01+08:00", "duration_ms": 883001, "first_token_ms": 8039, "model_requests": 38, "tokens": { "input_tokens": 3311021, "cached_input_tokens": 3217536, "cache_write_input_tokens": 0, "output_tokens": 22013, "reasoning_output_tokens": 4886, "total_tokens": 3333034 }, "standard_usd": 5.253036, "standard_credits": 131.3259, "cost_parts": { "uncached_input": 0.9348500000000002, "cached_input": 3.2175359999999995, "output": 1.10065 }, "max_request_input": 110614, "cache_hit": 0.9717655067726844, "name": "GPT-6 Astra", "display": "B · GPT-6 Astra / xhigh", "time_display": "14:43", "quality_rank": 1, "memory": { "database_mode": "disabled", "summary_injected": false, "observed_history_memory_access": false, "observed_memory_skill_read": false, "ordinary_skills": [ "Sites building", "Sites hosting", "Computer use" ] }, "service_tier": "not recorded; normalized Standard cost only", "original_page": "scenes/B.html", "original_source_zip": "sources/B.zip", "source_review": "code-review/B.html" } ], "comparison": { "astra_faster_percent": 32.36460873008761, "astra_saved_seconds": 422.53, "sol_cheaper_percent": 82.49475541382165, "astra_cost_multiple": 5.712573709421756, "aggregate_standard_usd": 6.172592799999999, "aggregate_standard_credits": 154.31482 }, "quality": { "criteria": [ "主体建模与细节", "原场景材质、光照与环境动态", "动画、运动与连接逻辑", "代码结构、资源处理与故障恢复" ], "weighting": "四项同等考虑，不发布伪精确的百分制总分。价格、耗时、拍摄遮挡不计入质量名次。", "order": [ "B", "A" ], "close_group": [ "B", "A" ], "scope": "仅比较本轮两件具体成品。非盲评，每种配置一次；综合排序为判断，差距有限。时间和费用不参与质量排序，瞬时镜头遮挡单列。", "conclusion": "本轮综合质量：B · GPT-6 Astra / xhigh 略优于 A · GPT-6.1 Sol / xhigh。Sol 的海面层次、日落和页面功能更丰富；Astra 的车轮关系、减少动画设置与图形中断提示更稳妥。", "dimensions": [ { "criterion": "主体建模与细节", "judgment": "两者接近", "A": "猫脸、头盔条纹和扣带、车灯、后视镜、尾巴与脚踏细节完整。", "B": "面部比例清楚，奶白头盔、车体、后视镜和背包细节完整。" }, { "criterion": "原场景材质、光照与环境动态", "judgment": "Sol 更丰富", "A": "海面有深浅渐层、波纹、反光和泡沫；七只鸟、云和两艘帆船有动态，另有日落切换。", "B": "海浪、七只鸟、云和帆船有实际运动；灯塔和柔和浅色海面构成完整环境，缺少日落切换。" }, { "criterion": "动画、运动与连接逻辑", "judgment": "Astra 更准确", "A": "方向正确，暂停冻结全场景；轮胎外半径0.453，滚动公式用0.445，累计轮距偏差约1.80%。", "B": "方向和半径关系正确；海面、鸟、云、围巾与车轮均随暂停冻结，恢复、调速有效。" }, { "criterion": "代码结构、资源处理与故障恢复", "judgment": "各有取舍，Astra 的异常反馈更可靠", "A": "共用基础几何和材质，材质42个；具有初次加载错误提示。但减少动画时仍播放，运行后图形中断的错误层被隐藏。", "B": "基础几何与材质按助手函数组织；默认绘制调用较少，但172个材质仍可合并。减少动画时停播，图形中断会显示刷新提示。" } ], "findings": [ { "sample": "A", "severity": "minor", "title": "轮半径轻微不一致", "evidence": "实际外半径0.345+0.108=0.453；滚动除数0.445，相对程序记录的路程偏差1.79775%。", "lines": [ 165, 296 ] }, { "sample": "A", "severity": "minor", "title": "系统减少动画设置未阻止自动播放", "evidence": "减少动画环境下仍为paused=false；固定步进两秒后行驶0.01112km。", "lines": [ 5, 6, 279, 292 ] }, { "sample": "A", "severity": "moderate", "title": "图形中断后错误提示不可见", "evidence": "加载完成把loading.style.display设为none；图形中断只修改class，实测display仍为none。", "lines": [ 286, 316 ] }, { "sample": "B", "severity": "improvement", "title": "材质资源可进一步共享", "evidence": "材质172个，三角面助手会克隆材质；相同外观可考虑共用。此项是资源改进建议，不是运行失败。", "lines": [ 32, 41 ] } ], "shared_limitations": [ "均是直线路面与景物后移的骑行，没有真实弯道转向系统；这是实现复杂度描述，不作为违反原提示词。", "原默认竖屏在某些时刻会被前景棕榈遮住主体。该观察单列，不能用一个截帧代表全部视觉质量，也不用于本轮排名。", "两组都没有自动图形恢复的验证证据；图形中断后的处理需要刷新。", "同一xhigh名称不代表模型计算量相同；每种配置只有一次样本。" ] }, "memory_audit": { "scope": "只核查本轮原任务；A/B为公开编号，不公开本地会话ID或会话全文。", "samples": [ { "sample": "A", "model": "GPT-6.1 Sol", "effort": "xhigh", "database_mode": "disabled", "summary_injected": false, "observed_history_memory_access": false, "observed_memory_skill_read": false, "ordinary_skills": [ "Sites building", "Sites hosting", "Computer use" ] }, { "sample": "B", "model": "GPT-6 Astra", "effort": "xhigh", "database_mode": "disabled", "summary_injected": false, "observed_history_memory_access": false, "observed_memory_skill_read": false, "ordinary_skills": [ "Sites building", "Sites hosting", "Computer use" ] } ], "conclusion": "两组数据库memory_mode均为disabled，起始上下文没有记忆摘要，工具日志未见读取历史记忆、旧作品或离线制作记忆技能。保留了普通工具与Sites技能；这不等于清空全部技能或隔离整台机器。", "workflow": "两组都主动采用Sites建站、源码同步和发布流程；原提示词没有要求云托管。这些工作与失败重试包含在原任务实际耗时中。", "timing_note": "两组运行窗口部分重叠，共用本地机器与工具服务；工具和发布策略未冻结，因此耗时差不能只归因于模型。", "history_note": "旧实验有历史记忆输入，本轮独立列出，不并入旧实验的排序，也不能用跨轮差异单独估计记忆的因果作用。" }, "pricing": { "date": "2026-10-02", "basis": "Standard, short context, text-token equivalent; not an account invoice or actual included-limit deduction", "usd_per_million": { "gpt-6.1-sol": { "uncached_input": 2, "cached_input": 0.1, "output": 10 }, "gpt-6-astra": { "uncached_input": 10, "cached_input": 1, "output": 50 } }, "credits_per_million": { "gpt-6.1-sol": { "uncached_input": 50, "cached_input": 2.5, "output": 250 }, "gpt-6-astra": { "uncached_input": 250, "cached_input": 25, "output": 1250 } }, "formula": "((input - cached_input) * uncached_rate + cached_input * cached_rate + output * output_rate) / 1e6", "notes": [ "推理Token包含在输出中，不能额外相加计费。", "两组缓存写入均为0，单次输入都低于272K。", "原记录未保存service_tier；仅作统一Standard折算，不声称本次实际采用了Standard或Fast。", "Codex credits使用独立官方Token费率计算，不能等同实际订阅额度扣减。", "不计未记录的工具独立收费；本次追加测评用量不纳入原任务。" ], "sources": [ { "title": "GPT-6.1 Sol model pricing", "url": "https://developers.openai.com/api/docs/models/gpt-6.1-sol" }, { "title": "GPT-6 Astra model pricing", "url": "https://developers.openai.com/api/docs/models/gpt-6-astra" }, { "title": "Codex Standard credit rates", "url": "https://learn.chatgpt.com/docs/pricing" } ] } }