A flat map of word meanings: places, animals, people and food each form their own neighbourhood.

1:"$Sreact.fragment" 2:I[727410,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"ThemeProvider"] 3:I[339756,["/_next/static/immutable/chunks/3w26vu4e29rrp.js"],"default"] 4:I[837457,["/_next/static/immutable/chunks/3w26vu4e29rrp.js"],"default"] 5:I[522016,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js","/_next/static/immutable/chunks/2fooooyx49quv.js"],""] 6:I[605500,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js","/_next/static/immutable/chunks/2fooooyx49quv.js"],"Image"] 7:I[505014,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js","/_next/static/immutable/chunks/2fooooyx49quv.js"],"default"] 1a:I[563491,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js","/_next/static/immutable/chunks/2gysr2xnt1og4.js","/_next/static/immutable/chunks/1kdir584oadgr.js"],"default"] :HL["/_next/static/immutable/chunks/2uot71sop0fdh.css","style"] :HL["/_next/static/immutable/chunks/2xv7ke5b-0f2b.css","style"] :HL["/_next/static/immutable/media/96d4d083aca4efcb-s.p.3skmobxb_0_5s.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/_next/static/immutable/media/b5893709a5ec98ee-s.p.0srb2049sag5h.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/_next/static/immutable/media/c9f6ebf08ddd616b-s.p.1er88gr2sg4r_.woff2","font",{"crossOrigin":"","type":"font/woff2"}] 18:X 0:{"P":null,"c":["","library","how-chatgpt-works-tokens-predictions-hallucinations-e4da","learn"],"q":"","i":false,"f":[[["",{"children":["(marketing)",{"children":["library",{"children":[["slug","how-chatgpt-works-tokens-predictions-hallucinations-e4da","d",["by"]],{"children":["learn",{"children":["__PAGE__",{},"$undefined","$undefined",4096]},"$undefined","$undefined",4096]},"$undefined","$undefined",4096]},"$undefined","$undefined",4096]},"$undefined","$undefined",4096]},"$undefined","$undefined",4112],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/2uot71sop0fdh.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/2xv7ke5b-0f2b.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/immutable/chunks/2ku_gc784a5rj.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/immutable/chunks/2rwzwt6nxusuz.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/immutable/chunks/2vb1dl-5o1641.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"className":"h-full scroll-smooth varela_round_749bfbb3-module__Znln_a__variable rubik_275070c2-module__aKq2iq__variable jetbrains_mono_cb2c698e-module__OrkwFq__variable noto_sans_hebrew_5fe0af4b-module__r9vlRW__variable frank_ruhl_libre_d68c43be-module__Brj_Ua__variable amatic_sc_d9d0ba63-module__flrIAG__variable atkinson_hyperlegible_next_24999921-module__vmKGQq__variable","children":[["$","head",null,{"children":[["$","meta",null,{"name":"theme-color","content":"#ffffff"}],["$","script",null,{"dangerouslySetInnerHTML":{"__html":"(function(){function s(d){var m=document.querySelector('meta[name=\"theme-color\"]');if(m)m.setAttribute(\"content\",d?\"#0f1114\":\"#ffffff\");}try{var t=localStorage.getItem(\"theme\")||\"system\";var d=t===\"dark\"||(t===\"system\"&&window.matchMedia(\"(prefers-color-scheme: dark)\").matches);var r=document.documentElement;r.classList.remove(\"light\",\"dark\");r.classList.add(d?\"dark\":\"light\");r.style.colorScheme=d?\"dark\":\"light\";s(d);}catch(e){try{var p=window.matchMedia&&window.matchMedia(\"(prefers-color-scheme: dark)\").matches;var r2=document.documentElement;r2.classList.remove(\"light\",\"dark\");r2.classList.add(p?\"dark\":\"light\");r2.style.colorScheme=p?\"dark\":\"light\";s(p);}catch(e2){}}})();"}}]]}],["$","body",null,{"className":"min-h-full flex flex-col antialiased","children":[["$","$L2",null,{"children":[["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"relative flex min-h-svh flex-col items-center justify-center overflow-x-clip px-4 py-16 text-center","children":[["$","div",null,{"aria-hidden":true,"className":"pointer-events-none absolute inset-0 -z-10 overflow-hidden","children":[["$","div",null,{"className":"absolute inset-x-0 top-0 h-[560px] bg-gradient-to-b from-wash to-transparent"}],["$","div",null,{"className":"mesh-blob mesh-blob-primary -top-48 left-[8%] h-[34rem] w-[34rem]"}]]}],["$","$L5",null,{"href":"/","className":"mb-10 flex items-center gap-2.5 rounded-[14px]","children":[["$","$L6",null,{"src":"/logo-192.webp","alt":"Workshopy.io","width":192,"height":192,"priority":false,"unoptimized":true,"className":"shrink-0 size-10"}],["$","span",null,{"className":"font-display text-[22px] font-extrabold tracking-[-0.02em] text-heading","children":["Workshopy",["$","span",null,{"className":"text-primary","children":".io"}]]}]]}],["$","div",null,{"className":"mb-8 inline-flex items-center gap-2 rounded-[14px] border border-tint bg-wash px-4 py-1.5 text-sm text-deep","children":[["$","$L7",null,{"ref":"$undefined","iconNode":[["circle",{"cx":"12","cy":"12","r":"10","key":"1mglay"}],["path",{"d":"m16.24 7.76-1.804 5.411a2 2 0 0 1-1.265 1.265L7.76 16.24l1.804-5.411a2 2 0 0 1 1.265-1.265z","key":"9ktpf1"}]],"className":"lucide-compass h-4 w-4","aria-hidden":true}],["$","span",null,{"children":"Off the workshop map"}]]}],["$","p",null,{"aria-hidden":true,"className":"animate-pop font-display text-[96px] font-extrabold italic leading-none tracking-[-0.04em] text-primary sm:text-[128px]","children":"404"}],"$L8","$L9","$La","$Lb"]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}],"$Lc","$Ld"]}],"$Le","$Lf","$L10","$L11","$L12"]}]]}]]}],{"children":["$L13",{"children":["$L14",{"children":["$L15",{"children":["$L16",{"children":["$L17",{},null,false,null]},null,false,"$18"]},null,false,"$18"]},null,false,"$18"]},null,false,null]},null,false,null],"$L19",false]],"m":"$undefined","G":["$1a",["$L1b","$L1c"]],"S":false,"h":null,"r":"$undefined","s":"$undefined","a":"$undefined","l":"$undefined","p":"$undefined","d":"$undefined"} 1d:I[713354,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"Toaster"] 1e:I[935655,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"CookieConsent"] 1f:I[462647,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"GoogleOneTap"] 20:I[46569,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"PostHogAnalytics"] 21:I[540123,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"GoogleAdsRoute"] 22:I[302355,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"Analytics"] 23:I[957215,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js"],"SpeedInsights"] 25:I[897367,["/_next/static/immutable/chunks/3w26vu4e29rrp.js"],"OutletBoundary"] 26:"$Sreact.suspense" 28:I[897367,["/_next/static/immutable/chunks/3w26vu4e29rrp.js"],"ViewportBoundary"] 2a:I[897367,["/_next/static/immutable/chunks/3w26vu4e29rrp.js"],"MetadataBoundary"] 8:["$","h1",null,{"className":"mt-4 font-display text-3xl font-extrabold tracking-[-0.02em] sm:text-4xl","children":"This page could not be found"}] 9:["$","p",null,{"className":"mt-3 max-w-md text-base leading-relaxed text-muted-foreground sm:text-lg","children":"The link may be broken, or the page may have been moved or deleted. Check the address — or head back to familiar ground."}] a:["$","div",null,{"className":"mt-10 flex w-full flex-col items-center justify-center gap-3 sm:flex-row","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center justify-center gap-2 whitespace-nowrap transition-[background-color,border-color,color,box-shadow] duration-200 focus-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 bg-primary text-primary-foreground hover:bg-primary/90 h-[52px] rounded-[14px] px-6 text-base [&_svg]:size-5 w-full sm:w-auto flex items-center justify-center gap-2","children":["Back to home",["$","$L7",null,{"ref":"$undefined","iconNode":[["path",{"d":"M5 12h14","key":"1ays0h"}],["path",{"d":"m12 5 7 7-7 7","key":"xquz4c"}]],"className":"lucide-arrow-right h-4 w-4"}]],"ref":null}],["$","$L5",null,{"href":"/join","children":"Join a workshop with a code","className":"inline-flex items-center justify-center gap-2 whitespace-nowrap transition-[background-color,border-color,color,box-shadow] duration-200 focus-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 border border-soft bg-background text-heading hover:bg-wash h-[52px] rounded-[14px] px-6 text-base [&_svg]:size-5 w-full sm:w-auto","ref":null}]]}] b:["$","p",null,{"className":"mt-6 text-sm text-muted-foreground","children":"Looking for a session? Ask your host for the join link or code."}] c:["$","$L1d",null,{}] d:["$","$L1e",null,{}] e:["$","$L1f",null,{}] f:["$","$L20",null,{}] 10:["$","$L21",null,{}] 11:["$","$L22",null,{}] 12:["$","$L23",null,{}] 13:["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"relative flex min-h-svh flex-col items-center justify-center overflow-x-clip px-4 py-16 text-center","children":[["$","div",null,{"aria-hidden":true,"className":"pointer-events-none absolute inset-0 -z-10 overflow-hidden","children":[["$","div",null,{"className":"absolute inset-x-0 top-0 h-[560px] bg-gradient-to-b from-wash to-transparent"}],["$","div",null,{"className":"mesh-blob mesh-blob-primary -top-48 left-[8%] h-[34rem] w-[34rem]"}]]}],["$","$L5",null,{"href":"/","className":"mb-10 flex items-center gap-2.5 rounded-[14px]","children":[["$","$L6",null,{"src":"/logo-192.webp","alt":"Workshopy.io","width":192,"height":192,"priority":false,"unoptimized":true,"className":"shrink-0 size-10"}],["$","span",null,{"className":"font-display text-[22px] font-extrabold tracking-[-0.02em] text-heading","children":["Workshopy",["$","span",null,{"className":"text-primary","children":".io"}]]}]]}],["$","div",null,{"className":"mb-8 inline-flex items-center gap-2 rounded-[14px] border border-tint bg-wash px-4 py-1.5 text-sm text-deep","children":[["$","$L7",null,{"ref":"$undefined","iconNode":"$0:f:0:1:0:props:children:1:props:children:1:props:children:0:props:children:0:props:notFound:0:props:children:2:props:children:0:props:iconNode","className":"lucide-compass h-4 w-4","aria-hidden":true}],["$","span",null,{"children":"Off the workshop map"}]]}],["$","p",null,{"aria-hidden":true,"className":"animate-pop font-display text-[96px] font-extrabold italic leading-none tracking-[-0.04em] text-primary sm:text-[128px]","children":"404"}],["$","h1",null,{"className":"mt-4 font-display text-3xl font-extrabold tracking-[-0.02em] sm:text-4xl","children":"This page could not be found"}],["$","p",null,{"className":"mt-3 max-w-md text-base leading-relaxed text-muted-foreground sm:text-lg","children":"The link may be broken, or the page may have been moved or deleted. Check the address — or head back to familiar ground."}],["$","div",null,{"className":"mt-10 flex w-full flex-col items-center justify-center gap-3 sm:flex-row","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center justify-center gap-2 whitespace-nowrap transition-[background-color,border-color,color,box-shadow] duration-200 focus-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 bg-primary text-primary-foreground hover:bg-primary/90 h-[52px] rounded-[14px] px-6 text-base [&_svg]:size-5 w-full sm:w-auto flex items-center justify-center gap-2","children":["Back to home",["$","$L7",null,{"ref":"$undefined","iconNode":"$a:props:children:0:props:children:1:props:iconNode","className":"lucide-arrow-right h-4 w-4"}]],"ref":null}],["$","$L5",null,{"href":"/join","children":"Join a workshop with a code","className":"inline-flex items-center justify-center gap-2 whitespace-nowrap transition-[background-color,border-color,color,box-shadow] duration-200 focus-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 border border-soft bg-background text-heading hover:bg-wash h-[52px] rounded-[14px] px-6 text-base [&_svg]:size-5 w-full sm:w-auto","ref":null}]]}],["$","p",null,{"className":"mt-6 text-sm text-muted-foreground","children":"Looking for a session? Ask your host for the join link or code."}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]]}] 14:["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}] 15:["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}] 16:["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}] 17:["$","$1","c",{"children":["$L24",[["$","script","script-0",{"src":"/_next/static/immutable/chunks/2sulpkebhh777.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/immutable/chunks/0cvw3lrukf80a.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/immutable/chunks/125b1080s_9ix.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/immutable/chunks/1lqz0eoxh_-ew.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/_next/static/immutable/chunks/1cnp8e47fi31t.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/_next/static/immutable/chunks/1b_4ehadho-3s.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/_next/static/immutable/chunks/2k4rd6igj2pm7.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/_next/static/immutable/chunks/1bbhy_id80uo_.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/_next/static/immutable/chunks/41mu_yriof-4f.js","async":true,"nonce":"$undefined"}]],["$","$L25",null,{"children":["$","$26",null,{"name":"Next.MetadataOutlet","children":"$@27"}]}]]}] 19:["$","$1","h",{"children":[null,["$","$L28",null,{"children":"$L29"}],["$","div",null,{"hidden":true,"children":["$","$L2a",null,{"children":["$","$26",null,{"name":"Next.Metadata","children":"$L2b"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] 1b:["$","link","0",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/2uot71sop0fdh.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] 1c:["$","link","1",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/2xv7ke5b-0f2b.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] 18:C 29:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1, viewport-fit=cover"}]] 2c:I[137969,["/_next/static/immutable/chunks/2ku_gc784a5rj.js","/_next/static/immutable/chunks/2rwzwt6nxusuz.js","/_next/static/immutable/chunks/2vb1dl-5o1641.js","/_next/static/immutable/chunks/2sulpkebhh777.js","/_next/static/immutable/chunks/0cvw3lrukf80a.js","/_next/static/immutable/chunks/125b1080s_9ix.js","/_next/static/immutable/chunks/1lqz0eoxh_-ew.js","/_next/static/immutable/chunks/1cnp8e47fi31t.js","/_next/static/immutable/chunks/1b_4ehadho-3s.js","/_next/static/immutable/chunks/2k4rd6igj2pm7.js","/_next/static/immutable/chunks/1bbhy_id80uo_.js","/_next/static/immutable/chunks/41mu_yriof-4f.js"],"LearnPlayer"] 37:I[27201,["/_next/static/immutable/chunks/3w26vu4e29rrp.js"],"IconMark"] 2d:T1e69,Your phone's keyboard guesses your next word. ChatGPT does the same job — far better, and over and over again. **Try it:** tap a suggestion to grow the sentence. Then press **Autopilot**. ```interactive title="Autocomplete: pick the next word"

Your message

``` > **The big idea:** the machine never plans the whole sentence. It only ever picks **the next piece**. Every step in this lesson builds on that.2e:T1df5,A model can't read letters. First, text is cut into **tokens** — whole common words, pieces of rarer words, punctuation — and each token is swapped for an ID number. **Try it:** click the examples, then type your own name. ```interactive title="Tokenizer: watch text split into tokens"

Type anything:

A small dot (·) is a space that belongs to the token. The ID numbers here are made up; a real model has a fixed list of about 100,000 tokens.

``` - Common words are one token; rare or long words split into pieces. - In English, one token is about ¾ of a word. - The model sees **tokens, not letters** — that's why it can miscount the r's in *strawberry*. **Done when:** you've found a word that splits into 3 or more tokens.2f:T2334,Each token ID is turned into a long list of numbers called an **embedding** — its address on a giant map of meaning. Tokens used in similar ways end up as neighbours. Real maps have thousands of directions; this is a flat sketch. **Try it:** tap *cat*. Then press **king − man + woman**. ```interactive title="A map of meaning"

``` Directions on the map carry meaning too: *male → female*, *country → capital*. Nobody gave the model definitions — it learned **where words sit** from how people use them.30:T2010,The same word can mean different things: a *river* bank isn't a *money* bank. So before guessing, each token **looks back at the earlier tokens** and borrows meaning from the ones that matter. This is called **attention**. **Try it:** switch between the four sentences and watch the outlined last word. ```interactive title="Attention: which earlier words matter"

``` A token can only look **back** — the words after it don't exist yet. Stack many layers of attention and you get a **Transformer**: the "T" in GPT.31:T15bb,After all that, the model gives **every token it knows** (about 100,000) a chance of coming next. Together the chances add up to 100%. **Try it:** compare the five prompts. Which ones have a single tall bar? ```interactive title="Next-token chances"

``` So the model never outputs *an answer* — it outputs **a chart like this**. Something still has to pick from it.32:T196d,Always taking the tallest bar is safe but dull. So the pick is a weighted roll of the dice, and **temperature** sets how bold the dice is. **Try it:** set the temperature to 0, then to 2. Press **Write 5 endings** each time. ```interactive title="Temperature dial"

The cat sat on the

Temperature: 1.0

``` - **Low** temperature: the top token wins almost every time — predictable. - **High** temperature: the small bars grow — creative, then nonsense (*moon*, *banana*). That's why the same question can get a different answer every time you ask.33:T2a8c,Now put the pieces together. To write an answer, the model runs the same loop again and again: ```mermaid flowchart LR A[Context window] --> B[Model] B --> C[Chance for every token] C --> D[Pick one] D -->|add it to the window| A ``` **Try it:** press **Play** and watch the haiku grow. Keep an eye on the start of the window. ```interactive title="The generation loop"

Context window — everything the model can see

Its top 3 guesses for the next token

The answer so far

``` - Each new token is fed back in. The model can't go back and fix an earlier word. - The **context window** — your messages plus its answer so far — is its only working memory. It has a size limit, and whatever falls out is forgotten. - A special **end** token tells it to stop.34:T2ab4,Where do the chances come from? From **training**: the model reads a huge amount of text and plays one game over and over — hide the next token, guess it, check, and adjust its numbers to guess a little better. **Try it:** press **Read 1 sentence** a few times, then **Read 50 more**. ```interactive title="Training a tiny model"

The model is about to read: “the cat ___”

It starts out with bad guesses — it thinks cats usually fly.

What comes after “the cat”?

Error (lower is better)

``` - At first its guesses are bad (it thinks cats fly). Each sentence pulls them toward the real text, and the error falls. - The error never reaches zero: language is never fully predictable. - A real LLM adjusts **billions of numbers** (its *parameters*) over **trillions of tokens**. Then people rate its answers to teach it to act as a helpful assistant.35:T2275,The model writes the most **plausible-sounding** next token — even when it has no idea. When it knows, the chart has one tall bar. When it doesn't, the bars are flat… but the pick still happens, and the sentence sounds just as sure. **Try it:** compare the two questions. Ask the second one two or three times. ```interactive title="Known fact vs. unknown fact"

The model's guesses for the first word of its answer

AI…

``` A confident, specific, made-up answer is a **hallucination**. Newer chatbots are trained to say "I don't know" more often, and some search the web — that helps, but it doesn't make them fully reliable. > **Rule:** check names, numbers, dates, quotes and links before you use them.36:T4d7,## A chatbot gives you a confident answer with an exact date. What should you assume? - [ ] It might be invented — check it - [ ] It's correct, because it sounds sure - [ ] It looked the date up in a database ## Why can the same prompt get two different answers? - [ ] The next token is picked with some randomness (temperature) - [ ] The model rewrites itself after every question - [ ] The model is broken ## What is the context window? - [ ] All the text the model can see right now — its working memory - [ ] Every website it read during training - [ ] The size of the chat box on your screen ## How did the model get its chances for each token? - [ ] By guessing the next token in huge amounts of text and adjusting when wrong - [ ] Programmers typed in a rule for every word - [ ] By copying a dictionary ## In "I need cash, so I'm going to the bank", what tells the model which "bank" is meant? - [ ] Attention to earlier words like "cash" - [ ] The length of the word - [ ] A spell-checker ## Which are good habits? (pick all) - [ ] Give context and an example of what you want - [ ] Check the facts and sources it gives you - [ ] Paste your passwords so it can help you faster - [ ] Trust it more when it sounds confident24:["$","$L2c",null,{"listing":{"id":"28516b20-e28e-4371-8324-76b3240e17bd","slug":"how-chatgpt-works-tokens-predictions-hallucinations-e4da","title":"How ChatGPT Works — Tokens, Predictions & Hallucinations","minutes":75},"author":{"name":"Workshopy"},"version":{"number":1,"steps":[{"kind":"content","title":"It's a very good autocomplete","position":1,"body_markdown":"$2d"},{"kind":"content","title":"Text becomes tokens","position":2,"body_markdown":"$2e"},{"kind":"content","title":"Tokens become a map of meaning","position":3,"body_markdown":"$2f"},{"kind":"content","title":"Attention: words look at each other","position":4,"body_markdown":"$30"},{"kind":"content","title":"The output: a chance for every token","position":5,"body_markdown":"$31"},{"kind":"content","title":"Temperature: how bold is the pick?","position":6,"body_markdown":"$32"},{"kind":"quiz","title":"Check: tokens and predictions","position":7,"body_markdown":"## What does a language model produce at each step?\n- [ ] A chance for every possible next token\n- [ ] A complete, planned answer\n- [ ] An answer copied from a database\n\n## Why can a model miscount the letters in \"strawberry\"?\n- [ ] It sees tokens (chunks), not single letters\n- [ ] It was never taught the alphabet\n- [ ] Counting needs an internet connection\n\n## Which pair sits closest together on a map of meaning?\n- [ ] cat and kitten\n- [ ] cat and car\n- [ ] cat and catalogue\n\n## You set the temperature to 0. What happens?\n- [ ] It always picks the most likely token\n- [ ] Its answers get more creative\n- [ ] It stops answering","question_types":["single","single","single","single"]},{"kind":"content","title":"One token at a time: the loop","position":8,"body_markdown":"$33"},{"kind":"content","title":"How it learned: guess, check, adjust","position":9,"body_markdown":"$34"},{"kind":"content","title":"Hallucinations: fluent isn't true","position":10,"body_markdown":"$35"},{"kind":"content","title":"Use it well","position":11,"body_markdown":"Everything you learned turns into five habits:\n\n| Because it… | So you… |\n|---|---|\n| predicts from the text it sees | give context: your goal, the audience, an example |\n| only knows what's in its window | paste the material you want it to use |\n| sounds sure even when guessing | check facts, numbers and sources |\n| picks with some randomness | ask again, compare, keep the best |\n| sends your text to a company's servers | never paste passwords or private data |\n\n## Try it (5 minutes)\n1. Ask an AI chatbot something you already know well. Was it right?\n2. Ask about something tiny or made up (a local event, an imaginary prize). Does it admit it doesn't know?\n3. Ask the same question twice and compare the answers.\n\n**Done when:** you've found one answer you would need to check."},{"kind":"quiz","title":"Final check","position":12,"body_markdown":"$36","question_types":["single","single","single","single","single","multiple"]}],"content_theme":null,"rtl":false},"next":{"slug":"intro-to-git-snapshots-branches-sharing-e775","title":"Intro to Git — Snapshots, Branches & Sharing"},"initialReading":{"theme":null,"size":"normal","font":"lesson","width":"normal"}}] 27:null 2b:[["$","title","0",{"children":"How ChatGPT Works — Tokens, Predictions & Hallucinations · Learn | Workshopy"}],["$","meta","1",{"name":"description","content":"A hands-on first look inside large language models (LLMs) like ChatGPT, Claude and Gemini — no maths, no code. Each step adds one idea, and each idea comes with"}],["$","link","2",{"rel":"manifest","href":"/manifest.webmanifest","crossOrigin":"$undefined"}],["$","meta","3",{"name":"robots","content":"noindex, follow"}],["$","meta","4",{"name":"google-site-verification","content":"1UK91FjdTGxYxvaHL523rvQ5C-PNM4ZI-9-A9Kd4PCM"}],["$","meta","5",{"property":"og:title","content":"Workshopy"}],["$","meta","6",{"property":"og:description","content":"Live workshop platform — run interactive workshops with real-time participant progress tracking."}],["$","meta","7",{"property":"og:url","content":"https://workshopy.io"}],["$","meta","8",{"property":"og:site_name","content":"Workshopy"}],["$","meta","9",{"property":"og:image","content":"https://workshopy.io/opengraph-image"}],["$","meta","10",{"property":"og:image:type","content":"image/png"}],["$","meta","11",{"property":"og:image:width","content":"1200"}],["$","meta","12",{"property":"og:image:height","content":"630"}],["$","meta","13",{"property":"og:image:alt","content":"Workshopy — live workshops written in plain Markdown, with every participant's progress in real time"}],["$","meta","14",{"property":"og:type","content":"website"}],["$","meta","15",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","16",{"name":"twitter:title","content":"Workshopy"}],["$","meta","17",{"name":"twitter:description","content":"Live workshop platform — run interactive workshops with real-time participant progress tracking."}],["$","meta","18",{"name":"twitter:image","content":"https://workshopy.io/opengraph-image"}],["$","meta","19",{"name":"twitter:image:alt","content":"Workshopy — live workshops written in plain Markdown, with every participant's progress in real time"}],["$","link","20",{"rel":"icon","href":"/favicon.ico?favicon.43vi01ib8snz1.ico","sizes":"32x32","type":"image/x-icon"}],["$","link","21",{"rel":"apple-touch-icon","href":"/apple-icon.png?apple-icon.0gns9w3k-u224.png","sizes":"180x180","type":"image/png"}],["$","$L37","22",{}]]