"""Build ../index.html from ../planted.csv and ../audit.csv, so no number on the page is typed by hand.
The Swedish page at sv/kan-en-laptop-granska-ai-research/ is a translation of this output: after a rebuild
that changes text or numbers, make the same change there by hand.
Usage: python3 build.py [YYYY-MM-DD] (after prepare.py; the date is the publication date)
"""
import csv, html, json, os, sys
from collections import Counter
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.join(HERE, "..")
rd = lambda name: list(csv.DictReader(open(os.path.join(ROOT, name), encoding="utf-8")))
PLANTED, AUDIT = rd("planted.csv"), rd("audit.csv")
DATE = sys.argv[1] if len(sys.argv) > 1 else "2026-10-25"
MONTHS = "January February March April May June July August September October November December".split()
LONGDATE = f"{MONTHS[int(DATE[5:7]) - 1]} {int(DATE[8:])}, {DATE[:4]}"
e = html.escape
REPO = "https://github.com/jbastide/receipts"
URL = "https://tools.copyfororiginals.com/can-a-laptop-check-ai-research/"
SV = "https://tools.copyfororiginals.com/sv/kan-en-laptop-granska-ai-research/"
# Numbers from our earlier study, reproduced by receipts (bench/model_study.py in the receipts repository)
REPRO_N, CONTROL_N, CONTROL_SWAP, CONTROL_CROSS = 593, 588, 0, 22
# ---- planted benchmark --------------------------------------------------------------------------
cases = [r for r in PLANTED if r["counted"] == "True"]
KINDS = ["number", "negation", "swap", "overclaim"]
KIND_NAME = {"number": "A number changed", "negation": "A verb negated", "swap": "Checked against another site’s page",
"overclaim": "An invented fact added"}
n_orig = sum(r["kind"] == "original" for r in cases)
n_kind = {k: sum(r["kind"] == k for r in cases) for k in KINDS}
n_def = sum(n_kind.values())
dropped = sum(r["counted"] == "False" for r in PLANTED)
def flagged(r, col):
return r[col] != "supported"
METHODS = [("MiniCheck (770M checker model)", lambda r: flagged(r, "minicheck"), ["minicheck"]),
("HHEM-2.1-Open (checker model)", lambda r: flagged(r, "hhem"), ["hhem"]),
("Gemma 4 12B as judge", lambda r: flagged(r, "judge"), ["judge"]),
("Gemma 4 12B judge, dense claims split", lambda r: flagged(r, "judge_split"), ["judge_split"]),
("MiniCheck or the judge flags it", lambda r: flagged(r, "minicheck") or flagged(r, "judge"), ["minicheck", "judge"])]
def seconds(cols):
"""Mean seconds per claim over every case; the split run reused whole-claim timings for short claims."""
return f"{sum(float(r[c + '_seconds'] or 0) for r in PLANTED for c in cols) / len(PLANTED):.1f}"
def score(fn):
passed = sum(not fn(r) for r in cases if r["kind"] == "original")
caught = {k: sum(fn(r) for r in cases if r["kind"] == k) for k in KINDS}
return passed, caught, sum(caught.values())
S = {name: score(fn) for name, fn, _ in METHODS}
J_PASS, J_KIND, J_ALL = S["Gemma 4 12B as judge"]
M_PASS, M_KIND, M_ALL = S["MiniCheck (770M checker model)"]
H_PASS, H_KIND, H_ALL = S["HHEM-2.1-Open (checker model)"]
def sort_state(r):
"""How the report files a claim with MiniCheck and the judge: pass, check by hand, or flagged."""
score_ = float(r["minicheck_score"] or 0)
j = r["judge"]
disagree = (j == "supported" and score_ < 0.5) or (j == "not_supported" and score_ >= 0.5)
return "hand" if disagree else ("clean" if j == "supported" else "flagged")
SORT = {grp: Counter(sort_state(r) for r in cases if (r["kind"] == "original") == grp) for grp in (True, False)}
def pct(a, b, dec=0):
return f"{100 * a / b:.{dec}f}%"
# ---- the real report --------------------------------------------------------------------------------
REV = Counter(r["review"] for r in AUDIT)
N_AUDIT = len(AUDIT)
N_BAD = REV["wrong"] + REV["unsupported"]
N_FIXED = sum(r["corrected"] == "True" for r in AUDIT)
N_QUOTES_BAD = sum(r["kind"] == "quote" for r in AUDIT)
# Facts about the real report and its correction, from the receipts run on the notes and the correction
# commit (see method/README.md); they are not in the two CSV files.
REPORT_CLAIMS, REPORT_SOURCES = 112, 87
QUOTE_FIXES = 7 # passages in quotation marks replaced by the source's own words
EXTRA_FIXES = 7 # errors found during the review outside the flagged items
REV_ROWS = [("correct", "Correct: the source says it"), ("wrong", "Wrong: the source says something different"),
("unsupported", "Not in any source the claim cites"), ("other_source", "Right, but on another page the claim cites"),
("author_analysis", "The report’s own reasoning, not a sourced fact"), ("cannot_check", "Source could not be read")]
# ---- FAQ ------------------------------------------------------------------------------------------
faq = [
("Can a small AI model on a laptop check whether a research report’s sources support it?",
f"Mostly, if it is used as a judge rather than as the last word. In a Copy for Originals test, Gemma 4 12B running on a MacBook Air with 16 GB of memory caught {J_ALL} of {n_def} claims we had deliberately damaged ({pct(J_ALL, n_def)}) and passed {J_PASS} of {n_orig} correct ones ({pct(J_PASS, n_orig)}), at about ten seconds a claim. Paired with a smaller checker model, {SORT[False]['clean']} of the {n_def} damaged claims got through without a flag."),
("How do you check the quotes in an AI research report?",
f"Fetch every cited page and look for each quoted passage in it, ignoring spacing, punctuation and case. That needs no AI model. Copy for Originals’ checker reproduced all {REPRO_N} quote grades from an earlier study this way, and when one word in each quote was replaced, none of {CONTROL_N} damaged quotes was accepted. A quote that is on the page can still fail to support the claim made with it, so the claim needs a second check."),
("What errors does an automated check find in real research?",
f"In our own published research notes on how AI shopping agents choose products, the checker flagged {N_AUDIT} items. A source-by-source review found {N_BAD} of them wrong or not in any cited source: paraphrases inside quotation marks, a range reported as +19.75 to +23.38 points that the paper gives as +13.0 to +37.9, a finding reported as no effect that the paper reports as statistically significant. {REV['correct']} were correct. The checker narrowed where to look; it did not decide."),
]
faq_ld = ",\n ".join(
'{"@type": "Question", "name": %s,\n "acceptedAnswer": {"@type": "Answer", "text": %s}}' % (json.dumps(q, ensure_ascii=False), json.dumps(a, ensure_ascii=False))
for q, a in faq)
faq_html = "\n".join(f"
{e(q)}
\n
{e(a)}
" for q, a in faq)
DESC = (f"Copy for Originals built an open-source checker for AI research reports that runs on a laptop. "
f"A 12B model caught {J_ALL} of {n_def} planted errors and passed {pct(J_PASS, n_orig)} of correct claims. "
f"In our own published research notes it led us to {N_BAD} wrong or unsupported claims. Data included.")
def method_rows():
out = []
for name, _, cols in METHODS:
p, k, a = S[name]
cells = [f"{p} of {n_orig}"] + [f"{k[x]} of {n_kind[x]}" for x in KINDS] + [f"{a} of {n_def} ({pct(a, n_def)})", seconds(cols)]
out.append(f'
Can a laptop check an AI research report? {n_def} planted errors and one real report
AI research tools now write a sourced report in minutes. Checking whether the sources say what the report says still takes a person most of a day. We built a checker that does the first pass on an ordinary laptop, with nothing sent to an AI company, and published it as open source. We tested it on {len(cases)} claims, {n_def} of them damaged on purpose, and then pointed it at research notes of our own that were already on this site. It caught most of the planted errors. In our own notes it led us to {N_BAD} claims that were wrong or not in their sources, and we have corrected them.
{J_ALL} of {n_def}planted errors caught by a 12B model on a 16 GB laptop ({pct(J_ALL, n_def)})
{J_PASS} of {n_orig}correct claims passed by the same model ({pct(J_PASS, n_orig)})
{N_BAD}wrong or unsupported claims found in our own published research notes
What did the test find?
Checking quotes needs no AI model. The checker fetches every cited page and looks for each quoted passage. It gave the same grade as our earlier study to all {REPRO_N} quotes, and when one word in each quote was replaced it accepted none of {CONTROL_N}.
A 12B model on a laptop is a usable judge. Gemma 4 12B, running on a MacBook Air with 16 GB of memory, caught {J_ALL} of {n_def} planted errors and passed {J_PASS} of {n_orig} correct claims, at about ten seconds a claim.
The smallest checker models are fast but not enough alone. MiniCheck caught {M_ALL} of {n_def} errors in under a second each, but rejected {n_orig - M_PASS} of {n_orig} correct claims. HHEM missed {n_kind['negation'] - H_KIND['negation']} of {n_kind['negation']} negated claims.
Used together, few errors slip through. When the report shows the judge’s verdict and sends disagreements to a person, {SORT[False]['clean']} of {n_def} planted errors passed without a flag. The price is that a reader checks about one claim in four by hand.
On a real report it narrows the search; it does not decide. In our research notes on how AI shopping agents choose products, it flagged {N_AUDIT} items. Checked against the sources one by one, {N_BAD} were wrong or not in any cited source, {REV['other_source']} were right but cited to the wrong page, and {REV['correct']} were correct.
The errors were the kind a careful reader misses. Paraphrases inside quotation marks, a range quoted as +19.75 to +23.38 points that the paper gives as +13.0 to +37.9, and a finding reported as no effect that the paper calls statistically significant.
What does the checker do?
It reads a report from ChatGPT, Claude, Gemini, Perplexity or an open-source research agent, fetches every page the report cites, and keeps a copy of each. Then it runs three checks.
Quotes. Is each quoted passage on the page it cites, word for word, edited, or missing? This is plain text matching, no model.
Links. Does each link work? For a dead link, the Wayback Machine tells a page that moved from one that may never have existed.
Support. Does the page back the claim? The checker picks the page sentences closest to the claim, a small checker model scores them, and a local chat model reads them and answers supported, partly or not supported. It names the sentences it relied on by number; it never writes a quotation itself. Long claims with several figures are split into single facts first, and each fact is checked on its own.
Everything runs on the user’s own machine or servers, with any model behind an OpenAI-compatible endpoint (llama.cpp, Ollama, vLLM, LM Studio). The code is at github.com/jbastide/receipts under the Apache 2.0 licence.
How well does it catch errors?
We started from {n_orig} claims in our earlier study of AI research runs that two graders had accepted and whose quotes were on the cited page. Each was checked as written, where it should pass, and in up to four damaged versions, where it should be flagged: one number changed, the first verb negated, the claim checked against a page from another website, or an invented fact added (“according to a 2019 survey of 4,000 law firms”). {dropped} negations of claims that were already negative made nonsense and are left out.
Correct claims passed and damaged claims caught, by method (Copy for Originals, {LONGDATE}; MacBook Air, Apple M5, 16 GB)
Method
Correct passed
""" + "".join(f'
{KIND_NAME[k]}
' for k in KINDS) + f"""
All errors caught
Seconds per claim
{method_rows()}
Splitting long claims into single facts made no difference here, because each planted error sits in one fact. It matters on real reports, where one sentence can carry five figures and the reader needs to know which one fails.
How the report files claims when MiniCheck and the judge are used together
Passed
Check by hand
Flagged
{n_orig} correct claims
{SORT[True]['clean']}
{SORT[True]['hand']}
{SORT[True]['flagged']}
{n_def} damaged claims
{SORT[False]['clean']}
{SORT[False]['hand']}
{SORT[False]['flagged']}
What did it find in a real report?
We ran it on the research notes behind our page How AI agents choose products: {REPORT_CLAIMS} claims citing {REPORT_SOURCES} sources, most of them academic papers. The checker flagged {N_QUOTES_BAD} quotes that were not on their pages and {N_AUDIT - N_QUOTES_BAD} facts with figures it could not confirm. Each flagged item was then checked against its source, reading the paper or page itself.
The {N_AUDIT} flagged items after review against the sources
Result
Items
{review_rows()}
So roughly one flag in four pointed at a real problem. That is the honest description of what a local checker does with dense research prose today: it cannot tell a missed sentence from a missing fact, but it turns a reading of {REPORT_SOURCES} sources into a list of {N_AUDIT} places to look. Some of what it found:
{QUOTE_FIXES} passages in quotation marks were not the source’s words. One paper says user profiles act “more like categorical switches or thresholds”; our notes had them “act as switches that override other cues”.
A defence that “left social-proof effects of +19.75 to +23.38 points intact” left them at +13.0 to +37.9 across the seven models in the paper’s table.
Two negative-framing cues reported as having no effect were, in the paper, “both statistically significantly effective”.
A crawl-to-referral ratio of 38,000:1 was right, but cited to a page with different, later numbers; it is Cloudflare’s own July 2025 figure (38,065.7 for Anthropic).
The published page had been checked before it went live and already used the sources’ wording. The research notes behind it had not. We corrected them: {N_FIXED} edits from the flagged items and {EXTRA_FIXES} more found while reviewing them. Afterwards every quote in the notes was on its cited page.
What are the limits?
The planted errors are mechanical. Real mistakes are subtler, so the catch rates above are a ceiling, not a promise.
On the real report we only know about the errors it flagged. Errors it passed as supported were not counted.
The judge reads the sentences that best match the claim, not the whole page. A claim drawn from a table or another part of the page can be marked unsupported.
One judge model, one machine, one run. Results with other models will differ; the checker is built so you can measure your own.
The review of the real report was done with the help of AI agents and spot-checked: every quote they relied on was matched against the source page by script.
How do I run it on my own reports?
On a Mac or Linux machine with uv installed and a local model server running:
git clone {REPO}
cd receipts
uv sync --extra support
uv run receipts check report.md --judge-url http://127.0.0.1:8080/v1
It writes a readable report, a JSON file and a CSV. The quote and link checks work without any model: uv run receipts check report.md.
Questions
{faq_html}
Data and method
planted.csv: all {len(PLANTED)} planted cases, the claim as checked, the page, and each method’s verdict and score.
audit.csv: the {N_AUDIT} items flagged in our research notes, the review result, the source passage and whether the notes were corrected.
Models: MiniCheck-Flan-T5-Large, HHEM-2.1-Open, Gemma 4 12B instruction-tuned (QAT, 4-bit) on llama.cpp. The benchmark script is bench/planted.py in the code repository.
"""
open(os.path.join(ROOT, "index.html"), "w", encoding="utf-8").write(page)
print(f"wrote index.html: judge {J_ALL}/{n_def} caught, {J_PASS}/{n_orig} passed; audit {N_BAD} bad of {N_AUDIT}")