attention.tex (3280B)
1 \begin{tikzpicture}[ 2 text shadow/.code args={[#1]#2at#3(#4)#5}{ 3 \pgfkeysalso{/tikz/.cd,#1}% 4 \foreach \angle in {0,5,...,359}{ 5 \node[#1,text=white] at ([shift={(\angle:.5pt)}] #4){#5}; 6 } 7 } 8 ] 9 \node (q) at (2.5, -1.5) {\(\vctr{q}\)}; 10 \node (o) at (2.5, 1.75) {\(\vctr{o}\)}; 11 \node (mk) at (0.12, -0.25) {}; 12 \node (mv) at (0.12, 0.55) {}; 13 \draw (0, -0.5) rectangle (2, 0); 14 \draw (0, 0.3) rectangle (2, 0.8); 15 \draw[dashdotted,Dark2-B,thick] (-0.25, 2.20) rectangle (3.2, -2.05); 16 \node (qh) at (2.5, -2.5) {query}; 17 \node (hi) at (2.5, 2.75) {output}; 18 19 \draw[arrow] (qh) -- (q); 20 \draw[arrow] (o) -- (hi); 21 22 \draw[decorate,decoration={brace,amplitude=5},xshift=0,yshift=0.5] (0,0.8) -- (2,0.8) node[black,midway] (memv) {}; 23 \node (ws) at (1, 1.15) {\scriptsize weighted sum}; 24 \draw[arrow] (ws) .. controls ++(90:0.75) and ++(180:0.5) .. (o); 25 \draw[decorate,decoration={brace,amplitude=5,mirror},xshift=0,yshift=-0.5] (0,-0.5) -- (2,-0.5) node[black,midway] (memv) {}; 26 \node (ip) at (1, -0.8) {\scriptsize inner product}; 27 \draw[arrow] (q) .. controls ++(180:0.5) and ++(270:0.75) .. (ip); 28 29 \node (s0) at (-6, -1) {\(\vctr{h}_0\)}; 30 \node (s1) at (-4.5, -1) {\(\vctr{h}_1\)}; 31 \node (sd) at (-3, -1) {\(\ldots\)}; 32 \node (sn) at (-1.5, -1) {\(\vctr{h}_\ell\)}; 33 34 \draw[arrow] (s0) -- (s1); 35 \draw[dashed,arrow] (s1) -- (sd); 36 \draw[dashed,arrow] (sd) -- (sn); 37 38 \node (x0) at (-6, -2) {\(\vctr{x}_0\)}; 39 \node (x1) at (-4.5, -2) {\(\vctr{x}_1\)}; 40 \node (xd) at (-3, -2) {\(\ldots\)}; 41 \node (xn) at (-1.5, -2) {\(\vctr{x}_\ell\)}; 42 43 \draw[arrow] (x0) -- (s0); 44 \draw[arrow] (x1) -- (s1); 45 \draw[arrow] (xn) -- (sn); 46 47 \draw[decorate,thick,color=Dark2-C,decoration={brace,amplitude=5},xshift=0,yshift=8] (-6.3,-1) -- (-1.2,-1) node[black,midway] (s) {}; 48 \node (henc) at (-3.75, -0.25) {memory}; 49 \draw (henc) .. controls ++(30:1) and ++(180:1) .. (-1.5,0.15); 50 \draw[arrow] (-1.5,0.15) .. controls ++(0:1) and ++(180:1) .. (mk); 51 \draw[arrow] (-1.5,0.15) .. controls ++(0:1) and ++(180:1) .. (mv); 52 53 \draw (0, 0.1) rectangle (2, 0.2); 54 \node at (2.6, 0.17) {\scriptsize softmax}; 55 \draw[fill=red!60] (0.4, 0.1) rectangle (0.5, 0.2); 56 \draw[fill=red!10] (0.5, 0.1) rectangle (0.6, 0.2); 57 \draw[fill=red!20] (1.2, 0.1) rectangle (1.3, 0.2); 58 \draw[fill=red!40] (1.3, 0.1) rectangle (1.4, 0.2); 59 \draw[fill=red!30] (1.4, 0.1) rectangle (1.5, 0.2); 60 \draw[fill=red!10] (1.8, 0.1) rectangle (1.9, 0.2); 61 62 \draw[fill=green!60] (0.4, 0.3) rectangle (0.5, 0.8); 63 \draw[fill=green!10] (0.5, 0.3) rectangle (0.6, 0.8); 64 \draw[fill=green!20] (1.2, 0.3) rectangle (1.3, 0.8); 65 \draw[fill=green!40] (1.3, 0.3) rectangle (1.4, 0.8); 66 \draw[fill=green!30] (1.4, 0.3) rectangle (1.5, 0.8); 67 \draw[fill=green!10] (1.8, 0.3) rectangle (1.9, 0.8); 68 69 \draw[fill=blue!60] (0.4, -0.5) rectangle (0.5, 0); 70 \draw[fill=blue!10] (0.5, -0.5) rectangle (0.6, 0); 71 \draw[fill=blue!20] (1.2, -0.5) rectangle (1.3, 0); 72 \draw[fill=blue!40] (1.3, -0.5) rectangle (1.4, 0); 73 \draw[fill=blue!30] (1.4, -0.5) rectangle (1.5, 0); 74 \draw[fill=blue!10] (1.8, -0.5) rectangle (1.9, 0); 75 76 \node[text shadow={[align=center,text width=3cm] at (1,0.55) {Value}}] at (1,0.55) {Value}; 77 \node[text shadow={[align=center,text width=3cm] at (1,-0.25) {Key}}] at (1,-0.25) {Key}; 78 \end{tikzpicture}%