Upload scandl2_demo.ipynb

#2
by minhgou - opened
Files changed (1) hide show
  1. scandl2_demo.ipynb +943 -0
scandl2_demo.ipynb ADDED
@@ -0,0 +1,943 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "id": "a8049e2d",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "%pip uninstall -y gradio gradio-client\n",
11
+ "%pip install \"huggingface-hub>=0.16.4,<0.18\"\n",
12
+ "%pip check"
13
+ ]
14
+ },
15
+ {
16
+ "cell_type": "markdown",
17
+ "id": "1d32e5f6",
18
+ "metadata": {},
19
+ "source": [
20
+ "# ScanDL2 Demo"
21
+ ]
22
+ },
23
+ {
24
+ "cell_type": "code",
25
+ "execution_count": 107,
26
+ "id": "73158dd6",
27
+ "metadata": {},
28
+ "outputs": [],
29
+ "source": [
30
+ "from ScanDL2 import ScanDL2\n",
31
+ "import torch\n",
32
+ "import sys\n",
33
+ "import torch\n",
34
+ "from unittest.mock import patch"
35
+ ]
36
+ },
37
+ {
38
+ "cell_type": "code",
39
+ "execution_count": 110,
40
+ "id": "6cb55d37",
41
+ "metadata": {},
42
+ "outputs": [],
43
+ "source": [
44
+ "sample_sentences = [\n",
45
+ " \"The quick brown fox jumps over the lazy dog.\",\n",
46
+ "]\n",
47
+ "sample_paragraphs = [\n",
48
+ " \"The sun was shining brightly in the clear blue sky. Birds chirped melodiously from the trees, filling the air with a cheerful tune. It was the perfect day for a walk in the park.\",\n",
49
+ " \"She had always loved the scent of fresh coffee in the morning. As she sipped her espresso, she glanced at the newspaper headlines. Another day, another adventure awaited her.\",\n",
50
+ " \"The conference room was filled with eager attendees. The keynote speaker took the stage, adjusting the microphone. With a confident smile, he began his presentation on the future of artificial intelligence.\",\n",
51
+ " \"They packed their bags and set off on their long-awaited road trip. The open road stretched ahead, inviting them to explore new places. With music playing softly in the background, they felt a sense of freedom like never before.\",\n",
52
+ "]"
53
+ ]
54
+ },
55
+ {
56
+ "cell_type": "code",
57
+ "execution_count": null,
58
+ "id": "762cf32a",
59
+ "metadata": {},
60
+ "outputs": [],
61
+ "source": [
62
+ "def display_scandl_scanpath(text, output, sample_index=0):\n",
63
+ " words = output[\"predicted_sp_words\"][sample_index]\n",
64
+ " positions = output[\"predicted_sp_ids\"][sample_index]\n",
65
+ " durations = output[\"predicted_fix_durs\"][sample_index]\n",
66
+ "\n",
67
+ " if not (len(words) == len(positions) == len(durations)):\n",
68
+ " raise ValueError(\"Word, position, and duration lengths do not match.\")\n",
69
+ "\n",
70
+ " rows = []\n",
71
+ "\n",
72
+ " for step in range(len(positions)):\n",
73
+ " word = str(words[step]).replace(\"|\", r\"\\|\")\n",
74
+ " rows.append(\n",
75
+ " f\"| {step + 1} | {word} | {positions[step]} \"\n",
76
+ " f\"| {float(durations[step]):.1f} |\"\n",
77
+ " )\n",
78
+ "\n",
79
+ " display(Markdown(\n",
80
+ " f\"**Input:** {text}\\n\\n\"\n",
81
+ " f\"**Scanpath:** {positions}\\n\\n\"\n",
82
+ " f\"**Fixated sequence:** {' → '.join(words)}\\n\\n\"\n",
83
+ " \"| Fixation | Word | Position | Duration (ms) |\\n\"\n",
84
+ " \"|---|---|---|---|\\n\"\n",
85
+ " + \"\\n\".join(rows)\n",
86
+ " ))\n",
87
+ "\n",
88
+ "def run_scandl_raw_text_input(model, text):\n",
89
+ " torch.manual_seed(0)\n",
90
+ "\n",
91
+ " with torch.no_grad():\n",
92
+ " return model(texts=[text])"
93
+ ]
94
+ },
95
+ {
96
+ "cell_type": "markdown",
97
+ "id": "34e2e22e",
98
+ "metadata": {},
99
+ "source": [
100
+ "## ScanDL2 — Sentences"
101
+ ]
102
+ },
103
+ {
104
+ "cell_type": "code",
105
+ "execution_count": 111,
106
+ "id": "9be2fa4a",
107
+ "metadata": {},
108
+ "outputs": [
109
+ {
110
+ "name": "stdout",
111
+ "output_type": "stream",
112
+ "text": [
113
+ "Logging to /var/folders/vg/2mjpcbv55m966znhbf9c5k580000gn/T/openai-2026-10-27-16-28-41-658594\n",
114
+ "### Loading ScanDL Diffusion Module ...\n",
115
+ "### Total number of parameters: 130855296\n",
116
+ "\t\t### ScanDL Module generates fixation locations ...\n"
117
+ ]
118
+ },
119
+ {
120
+ "name": "stderr",
121
+ "output_type": "stream",
122
+ "text": [
123
+ "1it [01:08, 68.80s/it]\n"
124
+ ]
125
+ },
126
+ {
127
+ "name": "stdout",
128
+ "output_type": "stream",
129
+ "text": [
130
+ "\t\t### FixDur Module generates fixation durations ...\n"
131
+ ]
132
+ },
133
+ {
134
+ "name": "stderr",
135
+ "output_type": "stream",
136
+ "text": [
137
+ "1it [00:00, 12.33it/s]"
138
+ ]
139
+ },
140
+ {
141
+ "name": "stdout",
142
+ "output_type": "stream",
143
+ "text": [
144
+ "fixdur original sn: ['The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog.']\n"
145
+ ]
146
+ },
147
+ {
148
+ "name": "stderr",
149
+ "output_type": "stream",
150
+ "text": [
151
+ "\n"
152
+ ]
153
+ },
154
+ {
155
+ "data": {
156
+ "text/markdown": [
157
+ "### Sentence 1"
158
+ ],
159
+ "text/plain": [
160
+ "<IPython.core.display.Markdown object>"
161
+ ]
162
+ },
163
+ "metadata": {},
164
+ "output_type": "display_data"
165
+ },
166
+ {
167
+ "data": {
168
+ "text/markdown": [
169
+ "**Input:** The quick brown fox jumps over the lazy dog.\n",
170
+ "\n",
171
+ "**Scanpath:** [1, 2, 3, 4, 5, 5, 6, 8, 8]\n",
172
+ "\n",
173
+ "**Fixated sequence:** The → quick → brown → fox → jumps → jumps → over → lazy → lazy\n",
174
+ "\n",
175
+ "| Fixation | Word | Position | Duration (ms) |\n",
176
+ "|---|---|---|---|\n",
177
+ "| 1 | The | 1 | 191.8 |\n",
178
+ "| 2 | quick | 2 | 188.5 |\n",
179
+ "| 3 | brown | 3 | 201.7 |\n",
180
+ "| 4 | fox | 4 | 207.9 |\n",
181
+ "| 5 | jumps | 5 | 219.3 |\n",
182
+ "| 6 | jumps | 5 | 229.0 |\n",
183
+ "| 7 | over | 6 | 235.7 |\n",
184
+ "| 8 | lazy | 8 | 221.9 |\n",
185
+ "| 9 | lazy | 8 | 173.3 |"
186
+ ],
187
+ "text/plain": [
188
+ "<IPython.core.display.Markdown object>"
189
+ ]
190
+ },
191
+ "metadata": {},
192
+ "output_type": "display_data"
193
+ }
194
+ ],
195
+ "source": [
196
+ "with patch.object(sys, \"argv\", [sys.argv[0]]):\n",
197
+ " scandl2_sent = ScanDL2(\n",
198
+ " text_type=\"sentence\",\n",
199
+ " bsz=2,\n",
200
+ " save=None,\n",
201
+ " filename=None,\n",
202
+ ")\n",
203
+ "scandl2_sent.eval()\n",
204
+ "\n",
205
+ "with torch.no_grad():\n",
206
+ " sentence_output = scandl2_sent(sample_sentences)\n",
207
+ "\n",
208
+ "for i, text in enumerate(sample_sentences):\n",
209
+ " display(Markdown(f\"### Sentence {i + 1}\"))\n",
210
+ " display_scandl_scanpath(text, sentence_output, sample_index=i)\n"
211
+ ]
212
+ },
213
+ {
214
+ "cell_type": "markdown",
215
+ "id": "2887484f",
216
+ "metadata": {},
217
+ "source": [
218
+ "## ScanDL2 — Paragraphs"
219
+ ]
220
+ },
221
+ {
222
+ "cell_type": "code",
223
+ "execution_count": 109,
224
+ "id": "6e695749",
225
+ "metadata": {},
226
+ "outputs": [
227
+ {
228
+ "name": "stdout",
229
+ "output_type": "stream",
230
+ "text": [
231
+ "Logging to /var/folders/vg/2mjpcbv55m966znhbf9c5k580000gn/T/openai-2026-10-27-16-08-03-314381\n",
232
+ "### Loading ScanDL Diffusion Module ...\n",
233
+ "### Total number of parameters: 131630560\n",
234
+ "\t\t### ScanDL Module generates fixation locations ...\n"
235
+ ]
236
+ },
237
+ {
238
+ "name": "stderr",
239
+ "output_type": "stream",
240
+ "text": [
241
+ "2it [11:09, 334.98s/it]\n"
242
+ ]
243
+ },
244
+ {
245
+ "name": "stdout",
246
+ "output_type": "stream",
247
+ "text": [
248
+ "\t\t### FixDur Module generates fixation durations ...\n"
249
+ ]
250
+ },
251
+ {
252
+ "name": "stderr",
253
+ "output_type": "stream",
254
+ "text": [
255
+ "1it [00:00, 2.03it/s]"
256
+ ]
257
+ },
258
+ {
259
+ "name": "stdout",
260
+ "output_type": "stream",
261
+ "text": [
262
+ "fixdur original sn: ['The', 'sun', 'was', 'shining', 'brightly', 'in', 'the', 'clear', 'blue', 'sky.', 'Birds', 'chirped', 'melodiously', 'from', 'the', 'trees,', 'filling', 'the', 'air', 'with', 'a', 'cheerful', 'tune.', 'It', 'was', 'the', 'perfect', 'day', 'for', 'a', 'walk', 'in', 'the', 'park.']\n",
263
+ "fixdur original sn: ['She', 'had', 'always', 'loved', 'the', 'scent', 'of', 'fresh', 'coffee', 'in', 'the', 'morning.', 'As', 'she', 'sipped', 'her', 'espresso,', 'she', 'glanced', 'at', 'the', 'newspaper', 'headlines.', 'Another', 'day,', 'another', 'adventure', 'awaited', 'her.']\n"
264
+ ]
265
+ },
266
+ {
267
+ "name": "stderr",
268
+ "output_type": "stream",
269
+ "text": [
270
+ "2it [00:00, 2.36it/s]"
271
+ ]
272
+ },
273
+ {
274
+ "name": "stdout",
275
+ "output_type": "stream",
276
+ "text": [
277
+ "fixdur original sn: ['The', 'conference', 'room', 'was', 'filled', 'with', 'eager', 'attendees.', 'The', 'keynote', 'speaker', 'took', 'the', 'stage,', 'adjusting', 'the', 'microphone.', 'With', 'a', 'confident', 'smile,', 'he', 'began', 'his', 'presentation', 'on', 'the', 'future', 'of', 'artificial', 'intelligence.']\n",
278
+ "fixdur original sn: ['They', 'packed', 'their', 'bags', 'and', 'set', 'off', 'on', 'their', 'long-awaited', 'road', 'trip.', 'The', 'open', 'road', 'stretched', 'ahead,', 'inviting', 'them', 'to', 'explore', 'new', 'places.', 'With', 'music', 'playing', 'softly', 'in', 'the', 'background,', 'they', 'felt', 'a', 'sense', 'of', 'freedom', 'like', 'never', 'before.']\n"
279
+ ]
280
+ },
281
+ {
282
+ "name": "stderr",
283
+ "output_type": "stream",
284
+ "text": [
285
+ "\n"
286
+ ]
287
+ },
288
+ {
289
+ "data": {
290
+ "text/markdown": [
291
+ "### Paragraph 1"
292
+ ],
293
+ "text/plain": [
294
+ "<IPython.core.display.Markdown object>"
295
+ ]
296
+ },
297
+ "metadata": {},
298
+ "output_type": "display_data"
299
+ },
300
+ {
301
+ "data": {
302
+ "text/markdown": [
303
+ "**Input:** The sun was shining brightly in the clear blue sky. Birds chirped melodiously from the trees, filling the air with a cheerful tune. It was the perfect day for a walk in the park.\n",
304
+ "\n",
305
+ "**Scanpath:** [2, 3, 4, 4, 5, 8, 7, 9, 11, 11, 13, 12, 12, 13, 14, 17, 17, 16, 18, 20, 22, 22, 23, 25, 27, 29, 28, 31, 31, 32, 34, 34]\n",
306
+ "\n",
307
+ "**Fixated sequence:** sun → was → shining → shining → brightly → clear → the → blue → Birds → Birds → melodiously → chirped → chirped → melodiously → from → filling → filling → trees, → the → with → cheerful → cheerful → tune. → was → perfect → for → day → walk → walk → in → park. → park.\n",
308
+ "\n",
309
+ "| Fixation | Word | Position | Duration (ms) |\n",
310
+ "|---|---|---|---|\n",
311
+ "| 1 | sun | 2 | 221.6 |\n",
312
+ "| 2 | was | 3 | 197.2 |\n",
313
+ "| 3 | shining | 4 | 212.7 |\n",
314
+ "| 4 | shining | 4 | 220.1 |\n",
315
+ "| 5 | brightly | 5 | 228.2 |\n",
316
+ "| 6 | clear | 8 | 226.2 |\n",
317
+ "| 7 | the | 7 | 224.6 |\n",
318
+ "| 8 | blue | 9 | 220.1 |\n",
319
+ "| 9 | Birds | 11 | 219.7 |\n",
320
+ "| 10 | Birds | 11 | 221.9 |\n",
321
+ "| 11 | melodiously | 13 | 227.9 |\n",
322
+ "| 12 | chirped | 12 | 231.2 |\n",
323
+ "| 13 | chirped | 12 | 221.8 |\n",
324
+ "| 14 | melodiously | 13 | 218.9 |\n",
325
+ "| 15 | from | 14 | 221.9 |\n",
326
+ "| 16 | filling | 17 | 230.9 |\n",
327
+ "| 17 | filling | 17 | 210.3 |\n",
328
+ "| 18 | trees, | 16 | 223.7 |\n",
329
+ "| 19 | the | 18 | 222.9 |\n",
330
+ "| 20 | with | 20 | 228.0 |\n",
331
+ "| 21 | cheerful | 22 | 223.6 |\n",
332
+ "| 22 | cheerful | 22 | 219.1 |\n",
333
+ "| 23 | tune. | 23 | 212.6 |\n",
334
+ "| 24 | was | 25 | 209.8 |\n",
335
+ "| 25 | perfect | 27 | 206.0 |\n",
336
+ "| 26 | for | 29 | 224.1 |\n",
337
+ "| 27 | day | 28 | 210.8 |\n",
338
+ "| 28 | walk | 31 | 196.1 |\n",
339
+ "| 29 | walk | 31 | 193.6 |\n",
340
+ "| 30 | in | 32 | 183.9 |\n",
341
+ "| 31 | park. | 34 | 189.7 |\n",
342
+ "| 32 | park. | 34 | 187.6 |"
343
+ ],
344
+ "text/plain": [
345
+ "<IPython.core.display.Markdown object>"
346
+ ]
347
+ },
348
+ "metadata": {},
349
+ "output_type": "display_data"
350
+ },
351
+ {
352
+ "data": {
353
+ "text/markdown": [
354
+ "### Paragraph 2"
355
+ ],
356
+ "text/plain": [
357
+ "<IPython.core.display.Markdown object>"
358
+ ]
359
+ },
360
+ "metadata": {},
361
+ "output_type": "display_data"
362
+ },
363
+ {
364
+ "data": {
365
+ "text/markdown": [
366
+ "**Input:** She had always loved the scent of fresh coffee in the morning. As she sipped her espresso, she glanced at the newspaper headlines. Another day, another adventure awaited her.\n",
367
+ "\n",
368
+ "**Scanpath:** [1, 2, 3, 4, 6, 8, 9, 11, 12, 15, 15, 15, 17, 19, 23, 24, 25, 27, 28, 28, 29]\n",
369
+ "\n",
370
+ "**Fixated sequence:** She → had → always → loved → scent → fresh → coffee → the → morning. → sipped → sipped → sipped → espresso, → glanced → headlines. → Another → day, → adventure → awaited → awaited → her.\n",
371
+ "\n",
372
+ "| Fixation | Word | Position | Duration (ms) |\n",
373
+ "|---|---|---|---|\n",
374
+ "| 1 | She | 1 | 220.6 |\n",
375
+ "| 2 | had | 2 | 195.1 |\n",
376
+ "| 3 | always | 3 | 211.5 |\n",
377
+ "| 4 | loved | 4 | 218.4 |\n",
378
+ "| 5 | scent | 6 | 226.8 |\n",
379
+ "| 6 | fresh | 8 | 226.2 |\n",
380
+ "| 7 | coffee | 9 | 223.3 |\n",
381
+ "| 8 | the | 11 | 219.0 |\n",
382
+ "| 9 | morning. | 12 | 217.6 |\n",
383
+ "| 10 | sipped | 15 | 219.6 |\n",
384
+ "| 11 | sipped | 15 | 225.9 |\n",
385
+ "| 12 | sipped | 15 | 227.9 |\n",
386
+ "| 13 | espresso, | 17 | 220.1 |\n",
387
+ "| 14 | glanced | 19 | 216.8 |\n",
388
+ "| 15 | headlines. | 23 | 219.4 |\n",
389
+ "| 16 | Another | 24 | 228.6 |\n",
390
+ "| 17 | day, | 25 | 207.4 |\n",
391
+ "| 18 | adventure | 27 | 222.9 |\n",
392
+ "| 19 | awaited | 28 | 220.6 |\n",
393
+ "| 20 | awaited | 28 | 227.6 |\n",
394
+ "| 21 | her. | 29 | 222.0 |"
395
+ ],
396
+ "text/plain": [
397
+ "<IPython.core.display.Markdown object>"
398
+ ]
399
+ },
400
+ "metadata": {},
401
+ "output_type": "display_data"
402
+ },
403
+ {
404
+ "data": {
405
+ "text/markdown": [
406
+ "### Paragraph 3"
407
+ ],
408
+ "text/plain": [
409
+ "<IPython.core.display.Markdown object>"
410
+ ]
411
+ },
412
+ "metadata": {},
413
+ "output_type": "display_data"
414
+ },
415
+ {
416
+ "data": {
417
+ "text/markdown": [
418
+ "**Input:** The conference room was filled with eager attendees. The keynote speaker took the stage, adjusting the microphone. With a confident smile, he began his presentation on the future of artificial intelligence.\n",
419
+ "\n",
420
+ "**Scanpath:** [1, 2, 3, 3, 5, 5, 7, 7, 8, 9, 10, 10, 12, 11, 14, 15, 17, 19, 21, 25, 23, 27, 28, 30, 31]\n",
421
+ "\n",
422
+ "**Fixated sequence:** The → conference → room → room → filled → filled → eager → eager → attendees. → The → keynote → keynote → took → speaker → stage, → adjusting → microphone. → a → smile, → presentation → began → the → future → artificial → intelligence.\n",
423
+ "\n",
424
+ "| Fixation | Word | Position | Duration (ms) |\n",
425
+ "|---|---|---|---|\n",
426
+ "| 1 | The | 1 | 220.4 |\n",
427
+ "| 2 | conference | 2 | 195.4 |\n",
428
+ "| 3 | room | 3 | 211.5 |\n",
429
+ "| 4 | room | 3 | 218.5 |\n",
430
+ "| 5 | filled | 5 | 226.9 |\n",
431
+ "| 6 | filled | 5 | 225.9 |\n",
432
+ "| 7 | eager | 7 | 223.4 |\n",
433
+ "| 8 | eager | 7 | 219.0 |\n",
434
+ "| 9 | attendees. | 8 | 217.8 |\n",
435
+ "| 10 | The | 9 | 219.9 |\n",
436
+ "| 11 | keynote | 10 | 226.0 |\n",
437
+ "| 12 | keynote | 10 | 228.4 |\n",
438
+ "| 13 | took | 12 | 220.2 |\n",
439
+ "| 14 | speaker | 11 | 217.0 |\n",
440
+ "| 15 | stage, | 14 | 219.7 |\n",
441
+ "| 16 | adjusting | 15 | 228.9 |\n",
442
+ "| 17 | microphone. | 17 | 207.9 |\n",
443
+ "| 18 | a | 19 | 222.7 |\n",
444
+ "| 19 | smile, | 21 | 220.9 |\n",
445
+ "| 20 | presentation | 25 | 227.4 |\n",
446
+ "| 21 | began | 23 | 222.1 |\n",
447
+ "| 22 | the | 27 | 218.0 |\n",
448
+ "| 23 | future | 28 | 210.6 |\n",
449
+ "| 24 | artificial | 30 | 208.1 |\n",
450
+ "| 25 | intelligence. | 31 | 203.7 |"
451
+ ],
452
+ "text/plain": [
453
+ "<IPython.core.display.Markdown object>"
454
+ ]
455
+ },
456
+ "metadata": {},
457
+ "output_type": "display_data"
458
+ },
459
+ {
460
+ "data": {
461
+ "text/markdown": [
462
+ "### Paragraph 4"
463
+ ],
464
+ "text/plain": [
465
+ "<IPython.core.display.Markdown object>"
466
+ ]
467
+ },
468
+ "metadata": {},
469
+ "output_type": "display_data"
470
+ },
471
+ {
472
+ "data": {
473
+ "text/markdown": [
474
+ "**Input:** They packed their bags and set off on their long-awaited road trip. The open road stretched ahead, inviting them to explore new places. With music playing softly in the background, they felt a sense of freedom like never before.\n",
475
+ "\n",
476
+ "**Scanpath:** [1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 16, 18, 17, 19, 19, 21, 21, 22, 22, 23, 25, 25, 27, 29, 30, 31, 32, 33, 36, 37, 36, 39]\n",
477
+ "\n",
478
+ "**Fixated sequence:** They → packed → their → bags → set → off → on → their → long-awaited → road → trip. → The → open → stretched → stretched → inviting → ahead, → them → them → explore → explore → new → new → places. → music → music → softly → the → background, → they → felt → a → freedom → like → freedom → before.\n",
479
+ "\n",
480
+ "| Fixation | Word | Position | Duration (ms) |\n",
481
+ "|---|---|---|---|\n",
482
+ "| 1 | They | 1 | 222.5 |\n",
483
+ "| 2 | packed | 2 | 198.3 |\n",
484
+ "| 3 | their | 3 | 213.7 |\n",
485
+ "| 4 | bags | 4 | 221.3 |\n",
486
+ "| 5 | set | 6 | 229.2 |\n",
487
+ "| 6 | off | 7 | 226.6 |\n",
488
+ "| 7 | on | 8 | 225.4 |\n",
489
+ "| 8 | their | 9 | 220.7 |\n",
490
+ "| 9 | long-awaited | 10 | 221.2 |\n",
491
+ "| 10 | road | 11 | 223.3 |\n",
492
+ "| 11 | trip. | 12 | 229.2 |\n",
493
+ "| 12 | The | 13 | 232.8 |\n",
494
+ "| 13 | open | 14 | 223.0 |\n",
495
+ "| 14 | stretched | 16 | 220.0 |\n",
496
+ "| 15 | stretched | 16 | 223.2 |\n",
497
+ "| 16 | inviting | 18 | 232.0 |\n",
498
+ "| 17 | ahead, | 17 | 211.8 |\n",
499
+ "| 18 | them | 19 | 224.5 |\n",
500
+ "| 19 | them | 19 | 224.2 |\n",
501
+ "| 20 | explore | 21 | 228.6 |\n",
502
+ "| 21 | explore | 21 | 224.7 |\n",
503
+ "| 22 | new | 22 | 220.0 |\n",
504
+ "| 23 | new | 22 | 214.0 |\n",
505
+ "| 24 | places. | 23 | 210.9 |\n",
506
+ "| 25 | music | 25 | 207.4 |\n",
507
+ "| 26 | music | 25 | 224.4 |\n",
508
+ "| 27 | softly | 27 | 212.3 |\n",
509
+ "| 28 | the | 29 | 198.9 |\n",
510
+ "| 29 | background, | 30 | 196.8 |\n",
511
+ "| 30 | they | 31 | 187.3 |\n",
512
+ "| 31 | felt | 32 | 192.9 |\n",
513
+ "| 32 | a | 33 | 191.2 |\n",
514
+ "| 33 | freedom | 36 | 174.0 |\n",
515
+ "| 34 | like | 37 | 186.6 |\n",
516
+ "| 35 | freedom | 36 | 182.3 |\n",
517
+ "| 36 | before. | 39 | 168.4 |"
518
+ ],
519
+ "text/plain": [
520
+ "<IPython.core.display.Markdown object>"
521
+ ]
522
+ },
523
+ "metadata": {},
524
+ "output_type": "display_data"
525
+ }
526
+ ],
527
+ "source": [
528
+ "with patch.object(sys, \"argv\", [sys.argv[0]]):\n",
529
+ " scandl2_par = ScanDL2(\n",
530
+ " text_type=\"paragraph\",\n",
531
+ " bsz=2,\n",
532
+ " save=None,\n",
533
+ " filename=None,\n",
534
+ ")\n",
535
+ "scandl2_par.eval()\n",
536
+ "\n",
537
+ "with torch.no_grad():\n",
538
+ " paragraph_output = scandl2_par(sample_paragraphs)\n",
539
+ "\n",
540
+ "for i, text in enumerate(sample_paragraphs):\n",
541
+ " display(Markdown(f\"### Paragraph {i + 1}\"))\n",
542
+ " display_scandl_scanpath(text, paragraph_output, sample_index=i)\n"
543
+ ]
544
+ },
545
+ {
546
+ "cell_type": "markdown",
547
+ "id": "1c44a7f5",
548
+ "metadata": {},
549
+ "source": [
550
+ "# Eyettention Demo"
551
+ ]
552
+ },
553
+ {
554
+ "cell_type": "markdown",
555
+ "id": "c33d6269",
556
+ "metadata": {},
557
+ "source": [
558
+ "### Chinese Language / BSC Dataset"
559
+ ]
560
+ },
561
+ {
562
+ "cell_type": "code",
563
+ "execution_count": null,
564
+ "id": "14a3e7ef",
565
+ "metadata": {},
566
+ "outputs": [],
567
+ "source": [
568
+ "from IPython.display import display, Markdown\n",
569
+ "import warnings\n",
570
+ "\n",
571
+ "warnings.filterwarnings(\"ignore\")\n",
572
+ "\n",
573
+ "def display_scanpath(text, scanpath, dataset=\"BSC\", previous_scanpath=None):\n",
574
+ " if dataset == \"BSC\":\n",
575
+ " units = [char for char in text if not char.isspace()]\n",
576
+ " elif dataset == \"celer\":\n",
577
+ " units = text.split()\n",
578
+ " else:\n",
579
+ " raise ValueError(\"dataset must be 'BSC' or 'celer'\")\n",
580
+ "\n",
581
+ " positions = scanpath[0].tolist()\n",
582
+ "\n",
583
+ " # Match the model's handling of an omitted start token.\n",
584
+ " if previous_scanpath is not None:\n",
585
+ " previous_scanpath = list(previous_scanpath)\n",
586
+ " if not previous_scanpath or previous_scanpath[0] != 0:\n",
587
+ " previous_scanpath = [0] + previous_scanpath\n",
588
+ "\n",
589
+ " rows = []\n",
590
+ " sequence = []\n",
591
+ "\n",
592
+ " for step, position in enumerate(positions):\n",
593
+ " if position == len(units) + 1: # End token\n",
594
+ " break\n",
595
+ " if position == 0: # Start token\n",
596
+ " continue\n",
597
+ "\n",
598
+ " unit = (\n",
599
+ " units[position - 1]\n",
600
+ " if 1 <= position <= len(units)\n",
601
+ " else \"?\"\n",
602
+ " )\n",
603
+ "\n",
604
+ " source = (\n",
605
+ " \"Observed\"\n",
606
+ " if previous_scanpath is not None and step < len(previous_scanpath)\n",
607
+ " else \"Generated\"\n",
608
+ " )\n",
609
+ "\n",
610
+ " sequence.append(unit)\n",
611
+ " rows.append(\n",
612
+ " f\"| {len(rows) + 1} | {unit} | {position} | {source} |\"\n",
613
+ " )\n",
614
+ "\n",
615
+ " display(Markdown(\n",
616
+ " f\"**Input:** {text}\\n\\n\"\n",
617
+ " f\"**Scanpath:** {scanpath.tolist()[0]}\\n\\n\"\n",
618
+ " f\"**Fixated sequence:** {' → '.join(sequence)}\\n\\n\"\n",
619
+ " \"| Fixation | Word / Character | Position | Source |\\n\"\n",
620
+ " \"|---|---|---|---|\\n\"\n",
621
+ " + \"\\n\".join(rows)\n",
622
+ " ))"
623
+ ]
624
+ },
625
+ {
626
+ "cell_type": "code",
627
+ "execution_count": null,
628
+ "id": "23163c57",
629
+ "metadata": {},
630
+ "outputs": [
631
+ {
632
+ "name": "stdout",
633
+ "output_type": "stream",
634
+ "text": [
635
+ "keeping Bert with pre-trained weights\n"
636
+ ]
637
+ },
638
+ {
639
+ "data": {
640
+ "text/markdown": [
641
+ "## Chinese Raw-text generation"
642
+ ],
643
+ "text/plain": [
644
+ "<IPython.core.display.Markdown object>"
645
+ ]
646
+ },
647
+ "metadata": {},
648
+ "output_type": "display_data"
649
+ },
650
+ {
651
+ "data": {
652
+ "text/markdown": [
653
+ "**Input:** 中国选手在男子滑雪比赛中有望蝉联冠军\n",
654
+ "\n",
655
+ "**Scanpath:** [0, 1, 1, 4, 3]\n",
656
+ "\n",
657
+ "**Fixated sequence:** 中 → 中 → 手 → 选\n",
658
+ "\n",
659
+ "| Fixation | Word / Character | Position | Source |\n",
660
+ "|---|---|---|---|\n",
661
+ "| 1 | 中 | 1 | Generated |\n",
662
+ "| 2 | 中 | 1 | Generated |\n",
663
+ "| 3 | 手 | 4 | Generated |\n",
664
+ "| 4 | 选 | 3 | Generated |"
665
+ ],
666
+ "text/plain": [
667
+ "<IPython.core.display.Markdown object>"
668
+ ]
669
+ },
670
+ "metadata": {},
671
+ "output_type": "display_data"
672
+ },
673
+ {
674
+ "data": {
675
+ "text/markdown": [
676
+ "---\n",
677
+ "\n",
678
+ "## Chinese Continuation from a previous scanpath"
679
+ ],
680
+ "text/plain": [
681
+ "<IPython.core.display.Markdown object>"
682
+ ]
683
+ },
684
+ "metadata": {},
685
+ "output_type": "display_data"
686
+ },
687
+ {
688
+ "data": {
689
+ "text/markdown": [
690
+ "**Input:** 中国选手在男子滑雪比赛中有望蝉联冠军\n",
691
+ "\n",
692
+ "**Scanpath:** [0, 1, 2, 3, 4, 4, 7, 9, 11]\n",
693
+ "\n",
694
+ "**Fixated sequence:** 中 → 国 → 选 → 手 → 手 → 子 → 雪 → 赛\n",
695
+ "\n",
696
+ "| Fixation | Word / Character | Position | Source |\n",
697
+ "|---|---|---|---|\n",
698
+ "| 1 | 中 | 1 | Observed |\n",
699
+ "| 2 | 国 | 2 | Observed |\n",
700
+ "| 3 | 选 | 3 | Observed |\n",
701
+ "| 4 | 手 | 4 | Generated |\n",
702
+ "| 5 | 手 | 4 | Generated |\n",
703
+ "| 6 | 子 | 7 | Generated |\n",
704
+ "| 7 | 雪 | 9 | Generated |\n",
705
+ "| 8 | 赛 | 11 | Generated |"
706
+ ],
707
+ "text/plain": [
708
+ "<IPython.core.display.Markdown object>"
709
+ ]
710
+ },
711
+ "metadata": {},
712
+ "output_type": "display_data"
713
+ }
714
+ ],
715
+ "source": [
716
+ "from Eyettention import EyettentionRawTextInference\n",
717
+ "\n",
718
+ "chinese_text = \"中国选手在男子滑雪比赛中有望蝉联冠军\"\n",
719
+ "chinese_previous_scanpath = [1,2,3]\n",
720
+ "\n",
721
+ "chinese_model = EyettentionRawTextInference(\n",
722
+ " checkpoint_path=\"Eyettention/results/BSC/Eyettention_chinese.pth\",\n",
723
+ " dataset=\"BSC\",\n",
724
+ " device=\"cpu\",\n",
725
+ ")\n",
726
+ "\n",
727
+ "def run_chinese_raw_text_input(model, text, max_pred_len=5):\n",
728
+ " torch.manual_seed(42)\n",
729
+ "\n",
730
+ " scanpath, _ = model.generate_from_chinese_text(\n",
731
+ " text=text,\n",
732
+ " max_pred_len=max_pred_len,\n",
733
+ " previous_scanpath=None\n",
734
+ " )\n",
735
+ " \n",
736
+ " return scanpath\n",
737
+ "\n",
738
+ "def run_chinese_previous_scanpath(model, text, max_pred_len=5, previous_scanpath=None):\n",
739
+ " torch.manual_seed(42)\n",
740
+ "\n",
741
+ " scanpath, _ = model.generate_from_chinese_text(\n",
742
+ " text=text,\n",
743
+ " max_pred_len=max_pred_len,\n",
744
+ " previous_scanpath= previous_scanpath\n",
745
+ " )\n",
746
+ "\n",
747
+ " return scanpath\n",
748
+ "\n",
749
+ "\n",
750
+ "display(Markdown(\"## Chinese Raw-text generation\"))\n",
751
+ "\n",
752
+ "raw_text = run_chinese_raw_text_input(chinese_model, chinese_text)\n",
753
+ "display_scanpath(chinese_text, scanpath=raw_text, dataset=\"BSC\")\n",
754
+ "\n",
755
+ "\n",
756
+ "display(Markdown(\"---\\n\\n## Chinese Continuation from a previous scanpath\"))\n",
757
+ "\n",
758
+ "\n",
759
+ "continuation = run_chinese_previous_scanpath(\n",
760
+ " chinese_model,\n",
761
+ " chinese_text,\n",
762
+ " previous_scanpath=chinese_previous_scanpath,\n",
763
+ ")\n",
764
+ "\n",
765
+ "display_scanpath(\n",
766
+ " chinese_text,\n",
767
+ " scanpath=continuation,\n",
768
+ " dataset=\"BSC\",\n",
769
+ " previous_scanpath=chinese_previous_scanpath,\n",
770
+ ")"
771
+ ]
772
+ },
773
+ {
774
+ "cell_type": "markdown",
775
+ "id": "399b583e",
776
+ "metadata": {},
777
+ "source": [
778
+ "### English Language / CELER Dataset"
779
+ ]
780
+ },
781
+ {
782
+ "cell_type": "code",
783
+ "execution_count": null,
784
+ "id": "f6ae615b",
785
+ "metadata": {},
786
+ "outputs": [
787
+ {
788
+ "name": "stdout",
789
+ "output_type": "stream",
790
+ "text": [
791
+ "keeping Bert with pre-trained weights\n"
792
+ ]
793
+ },
794
+ {
795
+ "data": {
796
+ "text/markdown": [
797
+ "## English Raw-text generation"
798
+ ],
799
+ "text/plain": [
800
+ "<IPython.core.display.Markdown object>"
801
+ ]
802
+ },
803
+ "metadata": {},
804
+ "output_type": "display_data"
805
+ },
806
+ {
807
+ "data": {
808
+ "text/markdown": [
809
+ "**Input:** The quick brown fox jumps over the lazy dog\n",
810
+ "\n",
811
+ "**Scanpath:** [0, 1, 4, 5, 8, 9, 3]\n",
812
+ "\n",
813
+ "**Fixated sequence:** The → fox → jumps → lazy → dog → brown\n",
814
+ "\n",
815
+ "| Fixation | Word / Character | Position | Source |\n",
816
+ "|---|---|---|---|\n",
817
+ "| 1 | The | 1 | Generated |\n",
818
+ "| 2 | fox | 4 | Generated |\n",
819
+ "| 3 | jumps | 5 | Generated |\n",
820
+ "| 4 | lazy | 8 | Generated |\n",
821
+ "| 5 | dog | 9 | Generated |\n",
822
+ "| 6 | brown | 3 | Generated |"
823
+ ],
824
+ "text/plain": [
825
+ "<IPython.core.display.Markdown object>"
826
+ ]
827
+ },
828
+ "metadata": {},
829
+ "output_type": "display_data"
830
+ },
831
+ {
832
+ "data": {
833
+ "text/markdown": [
834
+ "---\n",
835
+ "\n",
836
+ "## English Continuation from a previous scanpath"
837
+ ],
838
+ "text/plain": [
839
+ "<IPython.core.display.Markdown object>"
840
+ ]
841
+ },
842
+ "metadata": {},
843
+ "output_type": "display_data"
844
+ },
845
+ {
846
+ "data": {
847
+ "text/markdown": [
848
+ "**Input:** The quick brown fox jumps over the lazy dog\n",
849
+ "\n",
850
+ "**Scanpath:** [0, 1, 2, 3, 6, 9, 10, 4, 5]\n",
851
+ "\n",
852
+ "**Fixated sequence:** The → quick → brown → over → dog\n",
853
+ "\n",
854
+ "| Fixation | Word / Character | Position | Source |\n",
855
+ "|---|---|---|---|\n",
856
+ "| 1 | The | 1 | Observed |\n",
857
+ "| 2 | quick | 2 | Observed |\n",
858
+ "| 3 | brown | 3 | Observed |\n",
859
+ "| 4 | over | 6 | Generated |\n",
860
+ "| 5 | dog | 9 | Generated |"
861
+ ],
862
+ "text/plain": [
863
+ "<IPython.core.display.Markdown object>"
864
+ ]
865
+ },
866
+ "metadata": {},
867
+ "output_type": "display_data"
868
+ }
869
+ ],
870
+ "source": [
871
+ "english_text = \"The quick brown fox jumps over the lazy dog\"\n",
872
+ "english_previous_scanpath = [1,2,3]\n",
873
+ "\n",
874
+ "english_model = EyettentionRawTextInference(\n",
875
+ " checkpoint_path=\"Eyettention/results/CELER/Eyettention_english.pth\",\n",
876
+ " dataset=\"celer\",\n",
877
+ " device=\"cpu\",\n",
878
+ ")\n",
879
+ "\n",
880
+ "def run_english_raw_text_input(model, text, max_pred_len=7):\n",
881
+ " torch.manual_seed(42)\n",
882
+ "\n",
883
+ " scanpath, _ = model.generate_from_english_text(\n",
884
+ " text=text,\n",
885
+ " max_pred_len=max_pred_len,\n",
886
+ " previous_scanpath=None\n",
887
+ " )\n",
888
+ " \n",
889
+ " return scanpath\n",
890
+ "\n",
891
+ "def run_english_previous_scanpath(model, text, max_pred_len=5, previous_scanpath=None):\n",
892
+ " torch.manual_seed(42)\n",
893
+ "\n",
894
+ " scanpath, _ = model.generate_from_english_text(\n",
895
+ " text=text,\n",
896
+ " max_pred_len=max_pred_len,\n",
897
+ " previous_scanpath= previous_scanpath\n",
898
+ " )\n",
899
+ "\n",
900
+ " return scanpath\n",
901
+ "\n",
902
+ "\n",
903
+ "display(Markdown(\"## English Raw-text generation\"))\n",
904
+ "\n",
905
+ "raw_text = run_english_raw_text_input(english_model, english_text)\n",
906
+ "display_scanpath(english_text, scanpath=raw_text, dataset=\"celer\")\n",
907
+ "\n",
908
+ "\n",
909
+ "display(Markdown(\"---\\n\\n## English Continuation from a previous scanpath\"))\n",
910
+ "\n",
911
+ "\n",
912
+ "continuation = run_english_previous_scanpath(\n",
913
+ " english_model,\n",
914
+ " english_text,\n",
915
+ " previous_scanpath=english_previous_scanpath,\n",
916
+ ")\n",
917
+ "\n",
918
+ "display_scanpath(english_text, scanpath=continuation, dataset=\"celer\", previous_scanpath=english_previous_scanpath)"
919
+ ]
920
+ }
921
+ ],
922
+ "metadata": {
923
+ "kernelspec": {
924
+ "display_name": "scanpaths (3.10.7)",
925
+ "language": "python",
926
+ "name": "python3"
927
+ },
928
+ "language_info": {
929
+ "codemirror_mode": {
930
+ "name": "ipython",
931
+ "version": 3
932
+ },
933
+ "file_extension": ".py",
934
+ "mimetype": "text/x-python",
935
+ "name": "python",
936
+ "nbconvert_exporter": "python",
937
+ "pygments_lexer": "ipython3",
938
+ "version": "3.10.7"
939
+ }
940
+ },
941
+ "nbformat": 4,
942
+ "nbformat_minor": 5
943
+ }