{
 "manifest": "bench-v1 public comparability key",
 "generated_utc": "2026-09-03",
 "spec": {
  "protocol": "bench-v1 (locked 2026-09-02)",
  "tuple": {
   "core": "core-v1",
   "bench": "bench-v1"
  },
  "engine_git_sha": "3598916cd4e4b4fd639e3c8d31706453499a1100",
  "dataset": {
   "name": "LoCoMo-10 public release (Maharana et al., ACL 2024, CC BY-NC 4.0)",
   "scope": "all 10 public conversations, all five categories, n=1986",
   "note": "the full LoCoMo paper corpus is 50 conversations / 7512 questions; the public release used industry-wide as the vendor subset is these 10 conversations",
   "sha256": "sha256:79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4"
  },
  "judge_prompt_hashes": {
   "mem0-4o": "f2215542d36a40303e62c3fb3658d147c7a64bdcb2301d5558cd53fe582cc476",
   "mem0": "f2215542d36a40303e62c3fb3658d147c7a64bdcb2301d5558cd53fe582cc476",
   "mnemoverse": "703651c0f2afb274768a0c84d5d9239f759ba0cec529396a7e3d4ac968cc1f3d",
   "strict": "dd9f96f2330ec87f2e4d5357951af77ef8e6643917c6f0df48966bb8eadfbb23"
  },
  "judge_prompt_hash_rule": "sha256(system + newline + user_template), prompt_hash_version v1; mnemoverse and strict as registered with the cells, mem0 and mem0-4o from the same function over the re-judging prompt (one template, two models)",
  "embedding": {
   "model": "text-embedding-3-small",
   "dim": 1536,
   "source": "server-reported"
  },
  "judges": {
   "order_generous_to_strict": [
    "mem0-4o",
    "mem0",
    "mnemoverse",
    "strict"
   ],
   "mem0-4o": {
    "model": "gpt-4o",
    "prompt": "Mem0's published judge prompt, verbatim",
    "prompt_sha256": "f2215542d36a40303e62c3fb3658d147c7a64bdcb2301d5558cd53fe582cc476",
    "prompt_source": "https://raw.githubusercontent.com/mem0ai/memory-benchmarks/4b61c5d/benchmarks/locomo/prompts.py",
    "prompt_note": "fetched 2026-06-03; rendered in the upstream no-evidence variant (evidence_* placeholders empty), question/answer/response filled per item",
    "grading": "any one correct item of a list counts; paraphrases count",
    "role": "generous — Mem0's prompt on GPT-4o; a judge-model sensitivity probe (Mem0's own committed results file pins its grader as GPT-5)"
   },
   "mem0": {
    "model": "gpt-5",
    "prompt": "Mem0's published judge prompt, verbatim",
    "prompt_sha256": "f2215542d36a40303e62c3fb3658d147c7a64bdcb2301d5558cd53fe582cc476",
    "prompt_source": "https://raw.githubusercontent.com/mem0ai/memory-benchmarks/4b61c5d/benchmarks/locomo/prompts.py",
    "prompt_note": "fetched 2026-06-03; rendered in the upstream no-evidence variant (evidence_* placeholders empty), question/answer/response filled per item",
    "grading": "same as mem0-4o",
    "role": "generous — Mem0's published grader as its committed results file pins it (their prompt, GPT-5); the faithful mirror of the 92.5 headline's grading"
   },
   "mnemoverse": {
    "model": "gpt-5-mini",
    "prompt": "in-house binary CORRECT/WRONG",
    "prompt_sha256": "703651c0f2afb274768a0c84d5d9239f759ba0cec529396a7e3d4ac968cc1f3d",
    "prompt_source": "in-house, mnemoverse-benchmarks judges.py (repository private today)",
    "grading": "paraphrases and extra detail fine; contradictory fact or no answer is WRONG; strict on dates and entities",
    "role": "ours — the headline"
   },
   "strict": {
    "model": "gpt-5",
    "prompt": "adversarial audit",
    "prompt_sha256": "dd9f96f2330ec87f2e4d5357951af77ef8e6643917c6f0df48966bb8eadfbb23",
    "prompt_source": "in-house, mnemoverse-benchmarks judges.py (repository private today)",
    "grading": "partial coverage is WRONG; any wrong date or number is WRONG",
    "role": "strict — conservative floor"
   },
   "empty_answer_rule": "an empty answer counts 0 under every grader (ASYM-010 closed symmetrically)"
  },
  "reader": "gpt-5-mini",
  "k": 10,
  "registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652"
 },
 "corpus_aggregates": {
  "generated_from_registry_sha256": "aab5f8e0aa7b449bbe0efe76d2c4d70a17e997719dc0eed5e7961bf68e103652",
  "runs": [
   {
    "run_id": "baseline-20260830_mnemoverse_http",
    "system": "mnemoverse_http",
    "n_questions": 1986,
    "judges": {
     "mnemoverse": {
      "value": 0.546828,
      "ci95": [
       0.524673,
       0.568983
      ]
     },
     "strict": {
      "value": 0.338369
     },
     "mem0": {
      "value": 0.59718,
      "n_scored": 1986,
      "source": "rejudge of the registered cells' saved answers; empty answers count 0 under every grader (ASYM-010)"
     },
     "mem0-4o": {
      "value": 0.656596,
      "n_scored": 1986,
      "source": "rejudge of the registered cells' saved answers; empty answers count 0 under every grader (ASYM-010)"
     }
    },
    "recall": {
     "recall_at_cell_k": {
      "value": 0.631596,
      "n_scored": 1536
     },
     "recall_delivered": {
      "value": 0.67461,
      "n_scored": 1536
     }
    }
   },
   {
    "run_id": "baseline-20260830_naked_cosine",
    "system": "naked_cosine",
    "n_questions": 1986,
    "judges": {
     "mnemoverse": {
      "value": 0.569486,
      "ci95": [
       0.547331,
       0.591138
      ]
     },
     "strict": {
      "value": 0.359517
     },
     "mem0": {
      "value": 0.619335,
      "n_scored": 1986,
      "source": "rejudge of the registered cells' saved answers; empty answers count 0 under every grader (ASYM-010)"
     },
     "mem0-4o": {
      "value": 0.685801,
      "n_scored": 1986,
      "source": "rejudge of the registered cells' saved answers; empty answers count 0 under every grader (ASYM-010)"
     }
    },
    "recall": {
     "recall_at_cell_k": {
      "value": 0.713763,
      "n_scored": 1536
     },
     "recall_delivered": {
      "value": 0.713763,
      "n_scored": 1536
     }
    }
   },
   {
    "run_id": "tunedB-20260901_mnemoverse_http",
    "system": "mnemoverse_http",
    "n_questions": 1986,
    "judges": {
     "mnemoverse": {
      "value": 0.582578,
      "ci95": [
       0.560423,
       0.60423
      ]
     },
     "strict": {
      "value": 0.370594
     },
     "mem0": {
      "value": 0.630916,
      "n_scored": 1986,
      "source": "rejudge of the registered cells' saved answers; empty answers count 0 under every grader (ASYM-010)"
     },
     "mem0-4o": {
      "value": 0.697885,
      "n_scored": 1986,
      "source": "rejudge of the registered cells' saved answers; empty answers count 0 under every grader (ASYM-010)"
     }
    },
    "recall": {
     "recall_at_cell_k": {
      "value": 0.693935,
      "n_scored": 1536
     },
     "recall_delivered": {
      "value": 0.741707,
      "n_scored": 1536
     }
    }
   }
  ],
  "paired_contrasts": [
   {
    "name": "tunedB_mnemoverse_http_vs_baseline_naked_cosine",
    "a": "tunedB-20260901_mnemoverse_http",
    "b": "baseline-20260830_naked_cosine",
    "n_paired_questions": 1986,
    "mnemoverse": {
     "diff": 0.013092,
     "ci95": [
      -0.006042,
      0.032226
     ],
     "frac_resamples_positive": 0.9049
    },
    "strict": {
     "diff": 0.011078
    }
   },
   {
    "name": "baseline_mnemoverse_http_vs_baseline_naked_cosine",
    "a": "baseline-20260830_mnemoverse_http",
    "b": "baseline-20260830_naked_cosine",
    "n_paired_questions": 1986,
    "mnemoverse": {
     "diff": -0.022659,
     "ci95": [
      -0.043303,
      -0.001511
     ],
     "frac_resamples_positive": 0.0162
    },
    "strict": {
     "diff": -0.021148
    }
   },
   {
    "name": "tunedB_vs_baseline_mnemoverse_http",
    "a": "tunedB-20260901_mnemoverse_http",
    "b": "baseline-20260830_mnemoverse_http",
    "n_paired_questions": 1986,
    "mnemoverse": {
     "diff": 0.03575,
     "ci95": [
      0.018127,
      0.053877
     ],
     "frac_resamples_positive": 1.0
    },
    "strict": {
     "diff": 0.032226
    }
   }
  ]
 },
 "runs": [
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-26_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-26",
   "n_questions": 199,
   "judge": {
    "mnemoverse": 0.537688,
    "strict": 0.301508
   },
   "recall_at_k": 0.646111,
   "recall_delivered": 0.674444,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_full1540_n199_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-30_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-30",
   "n_questions": 105,
   "judge": {
    "mnemoverse": 0.580952,
    "strict": 0.257143
   },
   "recall_at_k": 0.60679,
   "recall_delivered": 0.631481,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv30_n105_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-41_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-41",
   "n_questions": 193,
   "judge": {
    "mnemoverse": 0.57513,
    "strict": 0.373057
   },
   "recall_at_k": 0.701425,
   "recall_delivered": 0.73432,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv41_n193_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-42_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-42",
   "n_questions": 260,
   "judge": {
    "mnemoverse": 0.519231,
    "strict": 0.338462
   },
   "recall_at_k": 0.609959,
   "recall_delivered": 0.645461,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv42_n260_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-43_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-43",
   "n_questions": 242,
   "judge": {
    "mnemoverse": 0.586777,
    "strict": 0.367769
   },
   "recall_at_k": 0.677207,
   "recall_delivered": 0.71466,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv43_n242_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-44_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-44",
   "n_questions": 158,
   "judge": {
    "mnemoverse": 0.607595,
    "strict": 0.449367
   },
   "recall_at_k": 0.627778,
   "recall_delivered": 0.689431,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv44_n158_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-47_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-47",
   "n_questions": 190,
   "judge": {
    "mnemoverse": 0.494737,
    "strict": 0.347368
   },
   "recall_at_k": 0.617222,
   "recall_delivered": 0.657778,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv47_n190_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-48_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-48",
   "n_questions": 239,
   "judge": {
    "mnemoverse": 0.456067,
    "strict": 0.305439
   },
   "recall_at_k": 0.599127,
   "recall_delivered": 0.641187,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv48_n239_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-49_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-49",
   "n_questions": 196,
   "judge": {
    "mnemoverse": 0.576531,
    "strict": 0.306122
   },
   "recall_at_k": 0.586265,
   "recall_delivered": 0.661926,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv49_n196_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_mnemoverse_http_conv-50_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "mnemoverse_http",
   "conv": "conv-50",
   "n_questions": 204,
   "judge": {
    "mnemoverse": 0.578431,
    "strict": 0.323529
   },
   "recall_at_k": 0.639957,
   "recall_delivered": 0.688568,
   "source_cell": "results/full-locomo-20260830-stock/cell_mnemoverse_http_locomo_conv50_n204_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-26_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-26",
   "n_questions": 199,
   "judge": {
    "mnemoverse": 0.507538,
    "strict": 0.286432
   },
   "recall_at_k": 0.683889,
   "recall_delivered": 0.683889,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_full1540_n199_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-30_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-30",
   "n_questions": 105,
   "judge": {
    "mnemoverse": 0.609524,
    "strict": 0.27619
   },
   "recall_at_k": 0.704938,
   "recall_delivered": 0.704938,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv30_n105_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-41_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-41",
   "n_questions": 193,
   "judge": {
    "mnemoverse": 0.632124,
    "strict": 0.435233
   },
   "recall_at_k": 0.784978,
   "recall_delivered": 0.784978,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv41_n193_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-42_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-42",
   "n_questions": 260,
   "judge": {
    "mnemoverse": 0.480769,
    "strict": 0.284615
   },
   "recall_at_k": 0.673171,
   "recall_delivered": 0.673171,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv42_n260_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-43_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-43",
   "n_questions": 242,
   "judge": {
    "mnemoverse": 0.603306,
    "strict": 0.38843
   },
   "recall_at_k": 0.747285,
   "recall_delivered": 0.747285,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv43_n242_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-44_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-44",
   "n_questions": 158,
   "judge": {
    "mnemoverse": 0.613924,
    "strict": 0.436709
   },
   "recall_at_k": 0.699187,
   "recall_delivered": 0.699187,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv44_n158_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-47_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-47",
   "n_questions": 190,
   "judge": {
    "mnemoverse": 0.557895,
    "strict": 0.4
   },
   "recall_at_k": 0.733889,
   "recall_delivered": 0.733889,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv47_n190_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-48_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-48",
   "n_questions": 239,
   "judge": {
    "mnemoverse": 0.560669,
    "strict": 0.351464
   },
   "recall_at_k": 0.707853,
   "recall_delivered": 0.707853,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv48_n239_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-49_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-49",
   "n_questions": 196,
   "judge": {
    "mnemoverse": 0.596939,
    "strict": 0.367347
   },
   "recall_at_k": 0.660229,
   "recall_delivered": 0.660229,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv49_n196_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "baseline-20260830_naked_cosine_conv-50_k10",
   "paradigm": "regression",
   "date_utc": "20260830",
   "system": "naked_cosine",
   "conv": "conv-50",
   "n_questions": 204,
   "judge": {
    "mnemoverse": 0.583333,
    "strict": 0.367647
   },
   "recall_at_k": 0.744124,
   "recall_delivered": 0.744124,
   "source_cell": "results/full-locomo-20260830-stock/cell_naked_cosine_locomo_conv50_n204_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-26_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-26",
   "n_questions": 199,
   "judge": {
    "mnemoverse": 0.552764,
    "strict": 0.306533
   },
   "recall_at_k": 0.697222,
   "recall_delivered": 0.726111,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv26_n199_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-30_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-30",
   "n_questions": 105,
   "judge": {
    "mnemoverse": 0.647619,
    "strict": 0.304762
   },
   "recall_at_k": 0.707613,
   "recall_delivered": 0.749177,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv30_n105_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-41_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-41",
   "n_questions": 193,
   "judge": {
    "mnemoverse": 0.61658,
    "strict": 0.414508
   },
   "recall_at_k": 0.742982,
   "recall_delivered": 0.784101,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv41_n193_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-42_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-42",
   "n_questions": 260,
   "judge": {
    "mnemoverse": 0.55,
    "strict": 0.357692
   },
   "recall_at_k": 0.691756,
   "recall_delivered": 0.733541,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv42_n260_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-43_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-43",
   "n_questions": 242,
   "judge": {
    "mnemoverse": 0.628099,
    "strict": 0.38843
   },
   "recall_at_k": 0.735634,
   "recall_delivered": 0.772338,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv43_n242_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-44_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-44",
   "n_questions": 158,
   "judge": {
    "mnemoverse": 0.601266,
    "strict": 0.436709
   },
   "recall_at_k": 0.667344,
   "recall_delivered": 0.722493,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv44_n158_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-47_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-47",
   "n_questions": 190,
   "judge": {
    "mnemoverse": 0.542105,
    "strict": 0.394737
   },
   "recall_at_k": 0.704444,
   "recall_delivered": 0.733333,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv47_n190_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-48_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-48",
   "n_questions": 239,
   "judge": {
    "mnemoverse": 0.531381,
    "strict": 0.359833
   },
   "recall_at_k": 0.682112,
   "recall_delivered": 0.734555,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv48_n239_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-49_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-49",
   "n_questions": 196,
   "judge": {
    "mnemoverse": 0.607143,
    "strict": 0.331633
   },
   "recall_at_k": 0.639946,
   "recall_delivered": 0.714387,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv49_n196_k10.json",
   "validity_gate_pass": true
  },
  {
   "run_id": "tunedB-20260901_mnemoverse_http_conv-50_k10",
   "paradigm": "tuned-frontier",
   "date_utc": "20260901",
   "system": "mnemoverse_http",
   "conv": "conv-50",
   "n_questions": 204,
   "judge": {
    "mnemoverse": 0.593137,
    "strict": 0.397059
   },
   "recall_at_k": 0.670406,
   "recall_delivered": 0.746261,
   "source_cell": "results/full-locomo-20260901-tunedB/cell_mnemoverse_http_locomo_conv50_n204_k10.json",
   "validity_gate_pass": true
  }
 ],
 "raw_cells": "committed byte-exact in the mnemoverse-benchmarks repository (private today; each run above carries its cell path and the registry sha256 pins the whole set)",
 "vendor_runs": {
  "protocol": "identical to the registered runs above (reader gpt-5-mini, k=10); mnemoverse and strict were scored in the run, mem0 and mem0-4o are re-judgings of the same saved answers with the generous prompt (see spec.judges); weighted totals in results/competitors-full-20260902/README.md",
  "not_registered_note": "vendor rows are not in the bench-v1 registry (open protocol question); listed here for public traceability",
  "mem0_full_corpus": "n/a until 2026-10-01 — vendor free-tier search quota exhausted before measurement (dated in the README); the conv-26 requery cell below is the valid mem0 row",
  "runs": [
   {
    "system": "supermemory",
    "conv": "conv-30",
    "n_questions": 105,
    "judge": {
     "mnemoverse": 0.7238095238095238,
     "strict": 0.3238095238095238,
     "mem0": 0.695238,
     "mem0-4o": 0.780952
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv30_n105_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-41",
    "n_questions": 193,
    "judge": {
     "mnemoverse": 0.49740932642487046,
     "strict": 0.32642487046632124,
     "mem0": 0.57513,
     "mem0-4o": 0.632124
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv41_n193_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-42",
    "n_questions": 260,
    "judge": {
     "mnemoverse": 0.4307692307692308,
     "strict": 0.23461538461538461,
     "mem0": 0.511538,
     "mem0-4o": 0.542308
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv42_n260_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-43",
    "n_questions": 242,
    "judge": {
     "mnemoverse": 0.5082644628099173,
     "strict": 0.26859504132231404,
     "mem0": 0.53719,
     "mem0-4o": 0.561983
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv43_n242_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-44",
    "n_questions": 158,
    "judge": {
     "mnemoverse": 0.5316455696202531,
     "strict": 0.35443037974683544,
     "mem0": 0.556962,
     "mem0-4o": 0.607595
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv44_n158_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-47",
    "n_questions": 190,
    "judge": {
     "mnemoverse": 0.5210526315789473,
     "strict": 0.3263157894736842,
     "mem0": 0.552632,
     "mem0-4o": 0.589474
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv47_n190_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-48",
    "n_questions": 239,
    "judge": {
     "mnemoverse": 0.5857740585774058,
     "strict": 0.39330543933054396,
     "mem0": 0.631799,
     "mem0-4o": 0.677824
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv48_n239_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-49",
    "n_questions": 196,
    "judge": {
     "mnemoverse": 0.4846938775510204,
     "strict": 0.27040816326530615,
     "mem0": 0.535714,
     "mem0-4o": 0.612245
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv49_n196_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-50",
    "n_questions": 204,
    "judge": {
     "mnemoverse": 0.5637254901960784,
     "strict": 0.3333333333333333,
     "mem0": 0.54902,
     "mem0-4o": 0.583333
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/supermemory/cell_supermemory_locomo_conv50_n204_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-30",
    "n_questions": 105,
    "judge": {
     "mnemoverse": 0.26666666666666666,
     "strict": 0.09523809523809523,
     "mem0": 0.304762,
     "mem0-4o": 0.352381
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv30_n105_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-41",
    "n_questions": 193,
    "judge": {
     "mnemoverse": 0.2538860103626943,
     "strict": 0.11398963730569948,
     "mem0": 0.404145,
     "mem0-4o": 0.445596
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv41_n193_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-42",
    "n_questions": 260,
    "judge": {
     "mnemoverse": 0.25,
     "strict": 0.16153846153846155,
     "mem0": 0.338462,
     "mem0-4o": 0.338462
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv42_n260_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-43",
    "n_questions": 242,
    "judge": {
     "mnemoverse": 0.35537190082644626,
     "strict": 0.19008264462809918,
     "mem0": 0.396694,
     "mem0-4o": 0.454545
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv43_n242_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-44",
    "n_questions": 158,
    "judge": {
     "mnemoverse": 0.31645569620253167,
     "strict": 0.15822784810126583,
     "mem0": 0.348101,
     "mem0-4o": 0.398734
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv44_n158_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-47",
    "n_questions": 190,
    "judge": {
     "mnemoverse": 0.25263157894736843,
     "strict": 0.14210526315789473,
     "mem0": 0.347368,
     "mem0-4o": 0.415789
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv47_n190_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-48",
    "n_questions": 239,
    "judge": {
     "mnemoverse": 0.3054393305439331,
     "strict": 0.18828451882845187,
     "mem0": 0.439331,
     "mem0-4o": 0.447699
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260903",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": true,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv48_n239_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-49",
    "n_questions": 196,
    "judge": {
     "mnemoverse": 0.33163265306122447,
     "strict": 0.1989795918367347,
     "mem0": 0.387755,
     "mem0-4o": 0.484694
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv49_n196_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-50",
    "n_questions": 204,
    "judge": {
     "mnemoverse": 0.3480392156862745,
     "strict": 0.1715686274509804,
     "mem0": 0.377451,
     "mem0-4o": 0.416667
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "fullcorpus-20260902",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-full-20260902/zep/cell_zep_locomo_conv50_n204_k10.json"
   },
   {
    "system": "supermemory",
    "conv": "conv-26",
    "n_questions": 199,
    "judge": {
     "mnemoverse": 0.4472361809045226,
     "strict": 0.2663316582914573,
     "mem0": 0.552764,
     "mem0-4o": 0.592965
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260901",
    "cell_tag": "comp-conv26-20260901",
    "requery_of_existing_corpus": false,
    "source_cell": "results/competitors-conv26-2026-09/run-20260901/cell_supermemory_locomo_conv26_n199_k10.json"
   },
   {
    "system": "mem0_v3_cloud",
    "conv": "conv-26",
    "n_questions": 199,
    "judge": {
     "mnemoverse": 0.507537688442211,
     "strict": 0.2613065326633166,
     "mem0": 0.582915,
     "mem0-4o": 0.658291
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "comp-conv26-20260901",
    "requery_of_existing_corpus": true,
    "source_cell": "results/competitors-conv26-2026-09/run-20260902-requery/cell_mem0_v3_cloud_locomo_conv26_n199_k10.json"
   },
   {
    "system": "zep",
    "conv": "conv-26",
    "n_questions": 199,
    "judge": {
     "mnemoverse": 0.2613065326633166,
     "strict": 0.1306532663316583,
     "mem0": 0.366834,
     "mem0-4o": 0.452261
    },
    "recall": "n_a (vendor ids do not map to our dia-ids)",
    "date_utc": "20260902",
    "cell_tag": "comp-rerun-20260902",
    "requery_of_existing_corpus": true,
    "source_cell": "results/competitors-conv26-2026-09/run-20260902-requery/cell_zep_locomo_conv26_n199_k10.json"
   }
  ]
 }
}