{ "schema_version": "1.0", "generated_at": "2026-09-06T20:02:44.192Z", "n_cells": 57, "systems": [ "mem0_v3_cloud", "mnemoverse", "mnemoverse_engine", "mnemoverse_http", "mnemoverse_http_tunedb", "naked_cosine", "supermemory", "zep" ], "benchmarks": [ "beam", "locomo" ], "cells": [ { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k10", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k10.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5724, "judge_accuracy_by_judge": { "mnemoverse": 0.572368, "mem0": 0.684211, "mem0-4o": 0.710526, "strict": 0.256579 }, "cost_usd_total": 0.02, "wallclock_s": 1345, "git_sha": "6ee1326", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k100", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k100.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6645, "judge_accuracy_by_judge": { "mnemoverse": 0.664474, "mem0": 0.710526, "mem0-4o": 0.75, "strict": 0.276316 }, "cost_usd_total": 0.1271, "wallclock_s": 1290.73, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k20", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k20.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.625, "judge_accuracy_by_judge": { "mnemoverse": 0.625, "mem0": 0.717105, "mem0-4o": 0.736842, "strict": 0.282895 }, "cost_usd_total": 0.0317, "wallclock_s": 1199.71, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k200", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k200.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6645, "judge_accuracy_by_judge": { "mnemoverse": 0.664474, "mem0": 0.730263, "mem0-4o": 0.743421, "strict": 0.269737 }, "cost_usd_total": 0.1922, "wallclock_s": 1272.71, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k50", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k50.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.625, "judge_accuracy_by_judge": { "mnemoverse": 0.625, "mem0": 0.703947, "mem0-4o": 0.723684, "strict": 0.269737 }, "cost_usd_total": 0.0679, "wallclock_s": 1208.91, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_engine_beam_10m_conv0-9_n200_k200", "path": "cells/cell_mnemoverse_engine_beam_10m_conv0-9_n200_k200.json", "system": "mnemoverse_engine", "benchmark": "beam", "benchmark_scope": "10m_conv0-9", "benchmark_scope_n": 200, "role": "headline", "k": 200, "reader_model": "claude-sonnet-4-6", "primary_judge": "gpt-5", "headline_metric": 0.61, "judge_accuracy_by_judge": { "gpt-5": 0.61 }, "cost_usd_total": 1.7639, "wallclock_s": 108930.97, "git_sha": null, "eval_path": "beam_v1", "known_asymmetries": [ "BEAM-SINGLE-SEED", "BEAM-READER-MISMATCH", "BEAM-INPUT-TOKEN-ASYM" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "beam-10m-2026-05-26", "single-system", "single-seed", "engine-mode-in-process", "http-rerun-planned" ], "mode_disclosure_INFORMATIONAL": "engine-mode (in-process, offline BEAM harness) — NOT the production HTTP path; single-system capability measurement, fenced comparative:false by the viz sync; an HTTP re-run is planned.", "recall_provenance": "n_a", "recall_at_k": null, "comparative": false }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k10", "path": "cells/cell_supermemory_locomo_conv26_n152_k10.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.3421, "judge_accuracy_by_judge": { "mnemoverse": 0.342105, "mem0": 0.421053, "mem0-4o": 0.473684, "strict": 0.190789 }, "cost_usd_total": 0.0201, "wallclock_s": 1422.42, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k100", "path": "cells/cell_supermemory_locomo_conv26_n152_k100.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6382, "judge_accuracy_by_judge": { "mnemoverse": 0.638158, "mem0": 0.769737, "mem0-4o": 0.828947, "strict": 0.335526 }, "cost_usd_total": 0.0845, "wallclock_s": 1430.49, "git_sha": "fb67250", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k20", "path": "cells/cell_supermemory_locomo_conv26_n152_k20.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.4211, "judge_accuracy_by_judge": { "mnemoverse": 0.421053, "mem0": 0.532895, "mem0-4o": 0.546053, "strict": 0.236842 }, "cost_usd_total": 0.0439, "wallclock_s": 1455.25, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k200", "path": "cells/cell_supermemory_locomo_conv26_n152_k200.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6447, "judge_accuracy_by_judge": { "mnemoverse": 0.644737, "mem0": 0.796053, "mem0-4o": 0.835526, "strict": 0.348684 }, "cost_usd_total": 0.0843, "wallclock_s": 1422.59, "git_sha": "fb67250", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k50", "path": "cells/cell_supermemory_locomo_conv26_n152_k50.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6513, "judge_accuracy_by_judge": { "mnemoverse": 0.651316, "mem0": 0.789474, "mem0-4o": 0.822368, "strict": 0.348684 }, "cost_usd_total": 0.0843, "wallclock_s": 1563.56, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv26_n152_k10", "path": "cells/cell_zep_locomo_conv26_n152_k10.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.2171, "judge_accuracy_by_judge": { "mnemoverse": 0.217105, "mem0": 0.296053, "mem0-4o": 0.328947, "strict": 0.085526 }, "cost_usd_total": 0.0117, "wallclock_s": 1147.24, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv26_n152_k20", "path": "cells/cell_zep_locomo_conv26_n152_k20.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.3553, "judge_accuracy_by_judge": { "mnemoverse": 0.355263, "mem0": 0.480263, "mem0-4o": 0.546053, "strict": 0.125 }, "cost_usd_total": 0.0167, "wallclock_s": 1133.91, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv26_n152_k50", "path": "cells/cell_zep_locomo_conv26_n152_k50.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.4803, "judge_accuracy_by_judge": { "mnemoverse": 0.480263, "mem0": 0.638158, "mem0-4o": 0.684211, "strict": 0.190789 }, "cost_usd_total": 0.0318, "wallclock_s": 1202.61, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": [ "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n199_k10", "path": "cells/cell_mnemoverse_http_locomo_conv26_n199_k10.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5327, "judge_accuracy_by_judge": { "mnemoverse": 0.532663, "strict": 0.321608, "mem0": 0.693467, "mem0-4o": 0.723618 }, "cost_usd_total": 0.2275, "wallclock_s": 2180.32, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028", "ASYM-031", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n199_k100", "path": "cells/cell_mnemoverse_http_locomo_conv26_n199_k100.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6181, "judge_accuracy_by_judge": { "mnemoverse": 0.61809, "strict": 0.38191, "mem0": 0.748744, "mem0-4o": 0.763819 }, "cost_usd_total": 0.5387, "wallclock_s": 2916.49, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028", "ASYM-031", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n199_k20", "path": "cells/cell_mnemoverse_http_locomo_conv26_n199_k20.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5729, "judge_accuracy_by_judge": { "mnemoverse": 0.572864, "strict": 0.371859, "mem0": 0.668342, "mem0-4o": 0.723618 }, "cost_usd_total": 0.2716, "wallclock_s": 2204.18, "git_sha": "unknown", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028", "ASYM-031", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n199_k200", "path": "cells/cell_mnemoverse_http_locomo_conv26_n199_k200.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6482, "judge_accuracy_by_judge": { "mnemoverse": 0.648241, "strict": 0.41206, "mem0": 0.743719, "mem0-4o": 0.788945 }, "cost_usd_total": 0.8173, "wallclock_s": 3506.82, "git_sha": "unknown", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028", "ASYM-031", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n199_k50", "path": "cells/cell_mnemoverse_http_locomo_conv26_n199_k50.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6482, "judge_accuracy_by_judge": { "mnemoverse": 0.648241, "strict": 0.396985, "mem0": 0.728643, "mem0-4o": 0.758794 }, "cost_usd_total": 0.3772, "wallclock_s": 2524.56, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028", "ASYM-031", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n199_k10", "path": "cells/cell_naked_cosine_locomo_conv26_n199_k10.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5075, "judge_accuracy_by_judge": { "mnemoverse": 0.507538, "strict": 0.281407, "mem0": 0.60804, "mem0-4o": 0.688442 }, "cost_usd_total": 0.2054, "wallclock_s": 1610.37, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.618333, "10": 0.690556 }, "by_category_recall_at_k": { "temporal": { "n": 37, "recall_at_k": { "5": 0.837838, "10": 0.891892 } }, "open_domain": { "n": 11, "recall_at_k": { "5": 0.409091, "10": 0.409091 } }, "multi_hop": { "n": 32, "recall_at_k": { "5": 0.257812, "10": 0.440104 } }, "single_hop": { "n": 70, "recall_at_k": { "5": 0.7, "10": 0.742857 } } }, "n_questions": 199, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10 ], "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n199_k100", "path": "cells/cell_naked_cosine_locomo_conv26_n199_k100.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6281, "judge_accuracy_by_judge": { "mnemoverse": 0.628141, "strict": 0.346734, "mem0": 0.728643, "mem0-4o": 0.778894 }, "cost_usd_total": 0.4241, "wallclock_s": 1929.92, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.618333, "10": 0.690556, "20": 0.776667, "50": 0.896667, "100": 0.917222 }, "by_category_recall_at_k": { "temporal": { "n": 37, "recall_at_k": { "5": 0.837838, "10": 0.891892, "20": 0.945946, "50": 1, "100": 1 } }, "open_domain": { "n": 11, "recall_at_k": { "5": 0.409091, "10": 0.409091, "20": 0.545455, "50": 0.772727, "100": 0.863636 } }, "multi_hop": { "n": 32, "recall_at_k": { "5": 0.257812, "10": 0.440104, "20": 0.546875, "50": 0.78125, "100": 0.846354 } }, "single_hop": { "n": 70, "recall_at_k": { "5": 0.7, "10": 0.742857, "20": 0.828571, "50": 0.914286, "100": 0.914286 } } }, "n_questions": 199, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100 ], "cell_native_k": 100 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n199_k20", "path": "cells/cell_naked_cosine_locomo_conv26_n199_k20.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5729, "judge_accuracy_by_judge": { "mnemoverse": 0.572864, "strict": 0.331658, "mem0": 0.683417, "mem0-4o": 0.743719 }, "cost_usd_total": 0.2305, "wallclock_s": 1419.42, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.618333, "10": 0.690556, "20": 0.776667 }, "by_category_recall_at_k": { "temporal": { "n": 37, "recall_at_k": { "5": 0.837838, "10": 0.891892, "20": 0.945946 } }, "open_domain": { "n": 11, "recall_at_k": { "5": 0.409091, "10": 0.409091, "20": 0.545455 } }, "multi_hop": { "n": 32, "recall_at_k": { "5": 0.257812, "10": 0.440104, "20": 0.546875 } }, "single_hop": { "n": 70, "recall_at_k": { "5": 0.7, "10": 0.742857, "20": 0.828571 } } }, "n_questions": 199, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20 ], "cell_native_k": 20 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n199_k200", "path": "cells/cell_naked_cosine_locomo_conv26_n199_k200.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6432, "judge_accuracy_by_judge": { "mnemoverse": 0.643216, "strict": 0.386935, "mem0": 0.753769, "mem0-4o": 0.809045 }, "cost_usd_total": 0.665, "wallclock_s": 1944.14, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.618333, "10": 0.690556, "20": 0.776667, "50": 0.896667, "100": 0.913889, "200": 0.960556 }, "by_category_recall_at_k": { "temporal": { "n": 37, "recall_at_k": { "5": 0.837838, "10": 0.891892, "20": 0.945946, "50": 1, "100": 1, "200": 1 } }, "open_domain": { "n": 11, "recall_at_k": { "5": 0.409091, "10": 0.409091, "20": 0.545455, "50": 0.772727, "100": 0.863636, "200": 0.969697 } }, "multi_hop": { "n": 32, "recall_at_k": { "5": 0.257812, "10": 0.440104, "20": 0.546875, "50": 0.78125, "100": 0.830729, "200": 0.888021 } }, "single_hop": { "n": 70, "recall_at_k": { "5": 0.7, "10": 0.742857, "20": 0.828571, "50": 0.914286, "100": 0.914286, "200": 0.971429 } } }, "n_questions": 199, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100, 200 ], "cell_native_k": 200 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n199_k50", "path": "cells/cell_naked_cosine_locomo_conv26_n199_k50.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n199", "benchmark_scope_n": 199, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.598, "judge_accuracy_by_judge": { "mnemoverse": 0.59799, "strict": 0.361809, "mem0": 0.708543, "mem0-4o": 0.748744 }, "cost_usd_total": 0.304, "wallclock_s": 1998.31, "git_sha": "4889d76", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028", "ASYM-033" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "bench3small-2026-07-05", "wave1-clean-matrix" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.618333, "10": 0.690556, "20": 0.776667, "50": 0.896667 }, "by_category_recall_at_k": { "temporal": { "n": 37, "recall_at_k": { "5": 0.837838, "10": 0.891892, "20": 0.945946, "50": 1 } }, "open_domain": { "n": 11, "recall_at_k": { "5": 0.409091, "10": 0.409091, "20": 0.545455, "50": 0.772727 } }, "multi_hop": { "n": 32, "recall_at_k": { "5": 0.257812, "10": 0.440104, "20": 0.546875, "50": 0.78125 } }, "single_hop": { "n": 70, "recall_at_k": { "5": 0.7, "10": 0.742857, "20": 0.828571, "50": 0.914286 } } }, "n_questions": 199, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50 ], "cell_native_k": 50 }, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k10", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k10.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5947, "judge_accuracy_by_judge": { "mnemoverse": 0.594737, "strict": 0.373684, "mem0": 0.684211, "mem0-4o": 0.705263 }, "cost_usd_total": 0.1797, "wallclock_s": 2288.26, "git_sha": "5bd54cb", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k100", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k100.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6316, "judge_accuracy_by_judge": { "mnemoverse": 0.631579, "strict": 0.4, "mem0": 0.721053, "mem0-4o": 0.757895 }, "cost_usd_total": 0.304, "wallclock_s": 2330.49, "git_sha": "592b530", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k20", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k20.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6474, "judge_accuracy_by_judge": { "mnemoverse": 0.647368, "strict": 0.405263, "mem0": 0.731579, "mem0-4o": 0.752632 }, "cost_usd_total": 0.1934, "wallclock_s": 1640.96, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k200", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k200.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6737, "judge_accuracy_by_judge": { "mnemoverse": 0.673684, "strict": 0.415789, "mem0": 0.752632, "mem0-4o": 0.778947 }, "cost_usd_total": 0.4377, "wallclock_s": 1751.48, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k50", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k50.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6368, "judge_accuracy_by_judge": { "mnemoverse": 0.636842, "strict": 0.384211, "mem0": 0.715789, "mem0-4o": 0.747368 }, "cost_usd_total": 0.2352, "wallclock_s": 1676, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k10", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k10.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5895, "judge_accuracy_by_judge": { "mnemoverse": 0.589474, "strict": 0.457895, "mem0": 0.752632, "mem0-4o": 0.778947 }, "cost_usd_total": 0.213, "wallclock_s": 2320.37, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.22, "10": 0.257778 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.313725, "10": 0.338235 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.116667, "10": 0.183333 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.240964, "10": 0.283133 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10 ], "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k100", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k100.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6421, "judge_accuracy_by_judge": { "mnemoverse": 0.642105, "strict": 0.473684, "mem0": 0.784211, "mem0-4o": 0.821053 }, "cost_usd_total": 0.5161, "wallclock_s": 5247.8, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.212222, "10": 0.257778, "20": 0.286667, "50": 0.302222, "100": 0.311667 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0, "50": 0, "100": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.279412, "10": 0.338235, "20": 0.352941, "50": 0.362745, "100": 0.362745 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.158333, "20": 0.2, "50": 0.2, "100": 0.270833 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.23494, "10": 0.289157, "20": 0.325301, "50": 0.349398, "100": 0.349398 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100 ], "cell_native_k": 100 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k20", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k20.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6105, "judge_accuracy_by_judge": { "mnemoverse": 0.610526, "strict": 0.436842, "mem0": 0.768421, "mem0-4o": 0.821053 }, "cost_usd_total": 0.2598, "wallclock_s": 3068.4, "git_sha": "592b530", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.216667, "10": 0.257778, "20": 0.283333 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.284314, "10": 0.338235, "20": 0.338235 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.183333, "20": 0.2 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.240964, "10": 0.283133, "20": 0.325301 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20 ], "cell_native_k": 20 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k200", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k200.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6737, "judge_accuracy_by_judge": { "mnemoverse": 0.673684, "strict": 0.489474, "mem0": 0.810526, "mem0-4o": 0.857895 }, "cost_usd_total": 0.7767, "wallclock_s": 6113.16, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.213333, "10": 0.253333, "20": 0.286667, "50": 0.3, "100": 0.310556, "200": 0.316667 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0, "50": 0, "100": 0, "200": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.284314, "10": 0.338235, "20": 0.352941, "50": 0.352941, "100": 0.362745, "200": 0.377451 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.175, "20": 0.2, "50": 0.2, "100": 0.2625, "200": 0.283333 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.23494, "10": 0.277108, "20": 0.325301, "50": 0.349398, "100": 0.349398, "200": 0.349398 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100, 200 ], "cell_native_k": 200 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k50", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k50.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6105, "judge_accuracy_by_judge": { "mnemoverse": 0.610526, "strict": 0.452632, "mem0": 0.763158, "mem0-4o": 0.805263 }, "cost_usd_total": 0.3593, "wallclock_s": 3791.89, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.213333, "10": 0.254444, "20": 0.286667, "50": 0.302222 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0, "50": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.269608, "10": 0.338235, "20": 0.352941, "50": 0.362745 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.183333, "20": 0.2, "50": 0.2 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.240964, "10": 0.277108, "20": 0.325301, "50": 0.349398 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50 ], "cell_native_k": 50 }, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k10", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k10.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5105, "judge_accuracy_by_judge": { "mnemoverse": 0.510526, "strict": 0.352632, "mem0": 0.657895, "mem0-4o": 0.721053 }, "cost_usd_total": 0.1979, "wallclock_s": 2112.5, "git_sha": "73083d4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k100", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k100.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6263, "judge_accuracy_by_judge": { "mnemoverse": 0.626316, "strict": 0.463158, "mem0": 0.763158, "mem0-4o": 0.815789 }, "cost_usd_total": 0.4579, "wallclock_s": 1866.93, "git_sha": "f6c536b", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k20", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k20.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5842, "judge_accuracy_by_judge": { "mnemoverse": 0.584211, "strict": 0.447368, "mem0": 0.726316, "mem0-4o": 0.794737 }, "cost_usd_total": 0.227, "wallclock_s": 1901.93, "git_sha": "73083d4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k200", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k200.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6526, "judge_accuracy_by_judge": { "mnemoverse": 0.652632, "strict": 0.463158, "mem0": 0.778947, "mem0-4o": 0.836842 }, "cost_usd_total": 0.7003, "wallclock_s": 2082.32, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k50", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k50.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6579, "judge_accuracy_by_judge": { "mnemoverse": 0.657895, "strict": 0.452632, "mem0": 0.8, "mem0-4o": 0.831579 }, "cost_usd_total": 0.3254, "wallclock_s": 1956.73, "git_sha": "73083d4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k10", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k10.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5421, "judge_accuracy_by_judge": { "mnemoverse": 0.542105, "strict": 0.405263, "mem0": 0.678947, "mem0-4o": 0.742105 }, "cost_usd_total": 0.1878, "wallclock_s": 1444.19, "git_sha": "19a16e4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10 ], "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k100", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k100.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6053, "judge_accuracy_by_judge": { "mnemoverse": 0.605263, "strict": 0.452632, "mem0": 0.805263, "mem0-4o": 0.852632 }, "cost_usd_total": 0.3739, "wallclock_s": 1383.08, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805, "50": 0.89, "100": 0.933889 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385, "50": 0.692308, "100": 0.692308 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922, "50": 0.872549, "100": 0.901961 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833, "50": 0.741667, "100": 0.870833 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614, "50": 0.963855, "100": 1 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100 ], "cell_native_k": 100 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k20", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k20.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5789, "judge_accuracy_by_judge": { "mnemoverse": 0.578947, "strict": 0.405263, "mem0": 0.736842, "mem0-4o": 0.789474 }, "cost_usd_total": 0.2093, "wallclock_s": 1583.91, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20 ], "cell_native_k": 20 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k200", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k200.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6684, "judge_accuracy_by_judge": { "mnemoverse": 0.668421, "strict": 0.457895, "mem0": 0.821053, "mem0-4o": 0.857895 }, "cost_usd_total": 0.5811, "wallclock_s": 1387.42, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805, "50": 0.89, "100": 0.933889, "200": 0.98 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385, "50": 0.692308, "100": 0.692308, "200": 0.923077 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922, "50": 0.872549, "100": 0.901961, "200": 0.97549 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833, "50": 0.741667, "100": 0.870833, "200": 0.941667 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614, "50": 0.963855, "100": 1, "200": 1 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100, 200 ], "cell_native_k": 200 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k50", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k50.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6316, "judge_accuracy_by_judge": { "mnemoverse": 0.631579, "strict": 0.442105, "mem0": 0.789474, "mem0-4o": 0.815789 }, "cost_usd_total": 0.271, "wallclock_s": 1420.82, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805, "50": 0.89 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385, "50": 0.692308 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922, "50": 0.872549 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833, "50": 0.741667 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614, "50": 0.963855 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50 ], "cell_native_k": 50 }, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k10", "path": "cells/cell_supermemory_locomo_conv47_n190_k10.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6316, "judge_accuracy_by_judge": { "mnemoverse": 0.631579, "strict": 0.378947, "mem0": 0.731579, "mem0-4o": 0.757895 }, "cost_usd_total": 0.1812, "wallclock_s": 2088.17, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k100", "path": "cells/cell_supermemory_locomo_conv47_n190_k100.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.7211, "judge_accuracy_by_judge": { "mnemoverse": 0.721053, "strict": 0.489474, "mem0": 0.794737, "mem0-4o": 0.8 }, "cost_usd_total": 0.3198, "wallclock_s": 2158.37, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k20", "path": "cells/cell_supermemory_locomo_conv47_n190_k20.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6895, "judge_accuracy_by_judge": { "mnemoverse": 0.689474, "strict": 0.415789, "mem0": 0.747368, "mem0-4o": 0.768421 }, "cost_usd_total": 0.1954, "wallclock_s": 1788.1, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k50", "path": "cells/cell_supermemory_locomo_conv47_n190_k50.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.7053, "judge_accuracy_by_judge": { "mnemoverse": 0.705263, "strict": 0.436842, "mem0": 0.784211, "mem0-4o": 0.794737 }, "cost_usd_total": 0.2405, "wallclock_s": 1753.42, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv47_n190_k10", "path": "cells/cell_zep_locomo_conv47_n190_k10.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.0895, "judge_accuracy_by_judge": { "mnemoverse": 0.089474, "strict": 0.068421, "mem0": 0.121053, "mem0-4o": 0.142105 }, "cost_usd_total": 0.1716, "wallclock_s": 1322.58, "git_sha": "6c8cd98", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv47_n190_k20", "path": "cells/cell_zep_locomo_conv47_n190_k20.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.1579, "judge_accuracy_by_judge": { "mnemoverse": 0.157895, "strict": 0.1, "mem0": 0.231579, "mem0-4o": 0.242105 }, "cost_usd_total": 0.1788, "wallclock_s": 1430.35, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv47_n190_k50", "path": "cells/cell_zep_locomo_conv47_n190_k50.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.2316, "judge_accuracy_by_judge": { "mnemoverse": 0.231579, "strict": 0.152632, "mem0": 0.326316, "mem0-4o": 0.357895 }, "cost_usd_total": 0.1972, "wallclock_s": 1737.36, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_full_n1986_k10", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "locomo_full_n1986", "benchmark_scope_n": 1986, "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.546828, "judge_accuracy_by_judge": { "mem0-4o": 0.656596, "mem0": 0.59718, "mnemoverse": 0.546828, "strict": 0.338369 }, "git_sha": "3598916cd4e4b4fd639e3c8d31706453499a1100", "tags": [ "bench-v1" ], "eval_path": "bench-v1 (locked 2026-09-02)", "raw_data_url": "https://mnemoverse.com/docs/bench-v1-manifest.json", "source_cells": [ "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_full1540_n199_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv30_n105_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv41_n193_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv42_n260_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv43_n242_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv44_n158_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv47_n190_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv48_n239_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv49_n196_k10.json", "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv50_n204_k10.json" ], "date_utc": "20260830", "registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652", "path": "cells/cell_mnemoverse_http_locomo_full_n1986_k10.json", "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "10": 0.631596 }, "n_questions": 1986, "n_questions_with_evidence_eval": 1536, "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_tunedb_locomo_full_n1986_k10", "system": "mnemoverse_http_tunedb", "benchmark": "locomo", "benchmark_scope": "locomo_full_n1986", "benchmark_scope_n": 1986, "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.582578, "judge_accuracy_by_judge": { "mem0-4o": 0.697885, "mem0": 0.630916, "mnemoverse": 0.582578, "strict": 0.370594 }, "git_sha": "3598916cd4e4b4fd639e3c8d31706453499a1100", "tags": [ "bench-v1", "tuned-frontier" ], "eval_path": "bench-v1 (locked 2026-09-02)", "raw_data_url": "https://mnemoverse.com/docs/bench-v1-manifest.json", "source_cells": [ "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv26_n199_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv30_n105_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv41_n193_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv42_n260_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv43_n242_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv44_n158_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv47_n190_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv48_n239_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv49_n196_k10.json", "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv50_n204_k10.json" ], "date_utc": "20260901", "registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652", "role": "ablation", "mode_disclosure_INFORMATIONAL": "Tuned-frontier configuration (2026-09-01) — an upper-bound / ceiling measurement of a tuned config, not the shipped default. The shipped-default headline is the baseline (stock) Mnemoverse row.", "path": "cells/cell_mnemoverse_http_tunedb_locomo_full_n1986_k10.json", "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "10": 0.693935 }, "n_questions": 1986, "n_questions_with_evidence_eval": 1536, "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_full_n1986_k10", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "locomo_full_n1986", "benchmark_scope_n": 1986, "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.569486, "judge_accuracy_by_judge": { "mem0-4o": 0.685801, "mem0": 0.619335, "mnemoverse": 0.569486, "strict": 0.359517 }, "git_sha": "3598916cd4e4b4fd639e3c8d31706453499a1100", "tags": [ "bench-v1" ], "eval_path": "bench-v1 (locked 2026-09-02)", "raw_data_url": "https://mnemoverse.com/docs/bench-v1-manifest.json", "source_cells": [ "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_full1540_n199_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv30_n105_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv41_n193_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv42_n260_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv43_n242_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv44_n158_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv47_n190_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv48_n239_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv49_n196_k10.json", "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv50_n204_k10.json" ], "date_utc": "20260830", "registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652", "path": "cells/cell_naked_cosine_locomo_full_n1986_k10.json", "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "10": 0.713763 }, "n_questions": 1986, "n_questions_with_evidence_eval": 1536, "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_supermemory_locomo_full_n1986_k10", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "locomo_full_n1986", "benchmark_scope_n": 1986, "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5181268882175226, "judge_accuracy_by_judge": { "mem0-4o": 0.6082577296072507, "mem0": 0.5629406027190332, "mnemoverse": 0.5181268882175226, "strict": 0.3066465256797583 }, "git_sha": null, "tags": [ "bench-v1", "vendor-not-registered" ], "eval_path": "bench-v1 (locked 2026-09-02)", "raw_data_url": "https://mnemoverse.com/docs/bench-v1-manifest.json", "source_cells": [ "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv30_n105_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv41_n193_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv42_n260_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv43_n242_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv44_n158_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv47_n190_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv48_n239_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv49_n196_k10.json", "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv50_n204_k10.json", "results/competitors-conv26-2026-09/run-20260901/cell_supermemory_locomo_conv26_n199_k10.json" ], "date_utc": "20260902", "registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652", "mode_disclosure_INFORMATIONAL": "vendor rows are not in the bench-v1 registry (open protocol question); listed here for public traceability", "path": "cells/cell_supermemory_locomo_full_n1986_k10.json", "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_full_n1986_k10", "system": "zep", "benchmark": "locomo", "benchmark_scope": "locomo_full_n1986", "benchmark_scope_n": 1986, "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.29556898288016115, "judge_accuracy_by_judge": { "mem0-4o": 0.42296073363544817, "mem0": 0.3756294058408863, "mnemoverse": 0.29556898288016115, "strict": 0.1596173212487412 }, "git_sha": null, "tags": [ "bench-v1", "vendor-not-registered" ], "eval_path": "bench-v1 (locked 2026-09-02)", "raw_data_url": "https://mnemoverse.com/docs/bench-v1-manifest.json", "source_cells": [ "results/competitors-full-20260902/zep/cell_zep_locomo_conv30_n105_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv41_n193_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv42_n260_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv43_n242_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv44_n158_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv47_n190_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv48_n239_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv49_n196_k10.json", "results/competitors-full-20260902/zep/cell_zep_locomo_conv50_n204_k10.json", "results/competitors-conv26-2026-09/run-20260902-requery/cell_zep_locomo_conv26_n199_k10.json" ], "date_utc": "20260903", "registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652", "mode_disclosure_INFORMATIONAL": "vendor rows are not in the bench-v1 registry (open protocol question); listed here for public traceability", "path": "cells/cell_zep_locomo_full_n1986_k10.json", "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_full_n1986_k10", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "locomo_full_n1986", "benchmark_scope_n": 199, "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.507537688442211, "judge_accuracy_by_judge": { "mem0-4o": 0.658291, "mem0": 0.582915, "mnemoverse": 0.507537688442211, "strict": 0.2613065326633166 }, "git_sha": null, "tags": [ "bench-v1", "vendor-not-registered", "partial-corpus" ], "eval_path": "bench-v1 (locked 2026-09-02)", "raw_data_url": "https://mnemoverse.com/docs/bench-v1-manifest.json", "source_cells": [ "results/competitors-conv26-2026-09/run-20260902-requery/cell_mem0_v3_cloud_locomo_conv26_n199_k10.json" ], "date_utc": "20260902", "registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652", "mode_disclosure_INFORMATIONAL": "vendor rows are not in the bench-v1 registry (open protocol question); listed here for public traceability n/a until 2026-10-01 — vendor free-tier search quota exhausted before measurement (dated in the README); the conv-26 requery cell below is the valid mem0 row", "path": "cells/cell_mem0_v3_cloud_locomo_full_n1986_k10.json", "recall_provenance": "n_a", "recall_at_k": null, "comparative": true } ], "system_labels": { "mem0_v3_cloud": "Mem0", "mnemoverse": "Mnemoverse", "mnemoverse_engine": "Mnemoverse", "mnemoverse_http": "Mnemoverse", "mnemoverse_http_tunedb": "mnemoverse_http_tunedb", "naked_cosine": "Naked cosine (floor)", "supermemory": "Supermemory", "zep": "Zep" }, "quarantine": { "render_contract": "Render as 'not measurable' cards per docs/RENDER_CONTRACT.md rule 1; never as zeros, never ranked. Superseded entries are NOT rendered at all — the good re-run (superseded_by_git_sha) is the live cell; this entry only stops a basename-keyed gate from excluding it.", "n_entries": 24, "excluded_cell_ids": [ "cell_mnemoverse_engine_locomo_conv26_n152_k10", "cell_mnemoverse_engine_locomo_conv26_n152_k100", "cell_mnemoverse_engine_locomo_conv26_n152_k20", "cell_mnemoverse_engine_locomo_conv26_n152_k200", "cell_mnemoverse_engine_locomo_conv26_n152_k50", "cell_mnemoverse_http_locomo_conv26_n152_k10", "cell_mnemoverse_http_locomo_conv26_n152_k100", "cell_mnemoverse_http_locomo_conv26_n152_k20", "cell_mnemoverse_http_locomo_conv26_n152_k200", "cell_mnemoverse_http_locomo_conv26_n152_k50", "cell_naked_cosine_locomo_conv26_n152_k10", "cell_naked_cosine_locomo_conv26_n152_k100", "cell_naked_cosine_locomo_conv26_n152_k20", "cell_naked_cosine_locomo_conv26_n152_k200", "cell_naked_cosine_locomo_conv26_n152_k50" ], "excluded_by_scope": { "conv26_n152": 15 } } }