import{s as sl,o as tl}from"../chunks/scheduler.37c15a92.js";import{S as nl,i as al,g as J,s as p,r,A as pl,h as y,f as s,c as o,j as Ke,u as m,x as w,k as Le,y as ol,a as n,v as M,t as u,b as Oe,d as c,w as b,p as el}from"../chunks/index.2bf4358c.js";import{C as j}from"../chunks/CodeBlock.4f5fc1ad.js";import{C as ll}from"../chunks/CourseFloatingBanner.06521e8b.js";import{F as il}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as Ne,E as ul}from"../chunks/EditOnGithub.8916dc7c.js";function cl(d){let a,i;return a=new ll({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_tf.ipynb"}]}}),{c(){r(a.$$.fragment)},l(t){m(a.$$.fragment,t)},m(t,T){M(a,t,T),i=!0},i(t){i||(c(a.$$.fragment,t),i=!0)},o(t){u(a.$$.fragment,t),i=!1},d(t){b(a,t)}}}function rl(d){let a,i;return a=new ll({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_pt.ipynb"}]}}),{c(){r(a.$$.fragment)},l(t){m(a.$$.fragment,t)},m(t,T){M(a,t,T),i=!0},i(t){i||(c(a.$$.fragment,t),i=!0)},o(t){u(a.$$.fragment,t),i=!1},d(t){b(a,t)}}}function ml(d){let a,i;return a=new j({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBc2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoKip0b2tlbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){r(a.$$.fragment)},l(t){m(a.$$.fragment,t)},m(t,T){M(a,t,T),i=!0},i(t){i||(c(a.$$.fragment,t),i=!0)},o(t){u(a.$$.fragment,t),i=!1},d(t){b(a,t)}}}function Ml(d){let a,i;return a=new j({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){r(a.$$.fragment)},l(t){m(a.$$.fragment,t)},m(t,T){M(a,t,T),i=!0},i(t){i||(c(a.$$.fragment,t),i=!0)},o(t){u(a.$$.fragment,t),i=!1},d(t){b(a,t)}}}function bl(d){let a,i,t,T,Z,te,g,ne,h,f,ee,V,Xe="在最后几节中，我们一直在尽最大努力手工完成大部分工作。我们探讨了标记化器的工作原理，并研究了标记化、到输入ID的转换、填充、截断和注意掩码。",ae,k,_e="然而，正如我们在第2节中所看到的，🤗 Transformers API可以通过一个高级函数为我们处理所有这些，我们将在这里深入讨论。当你直接在句子上调用标记器时，你会得到准备通过模型传递的输入",pe,$,oe,W,xe=`这里，<code>model_inputs</code>
变量包含模型良好运行所需的一切。对于DistilBERT，它包括输入 ID和注意力掩码(attention mask)。其他接受额外输入的模型也会有标记器对象的输出。`,ie,B,ze="正如我们将在下面的一些示例中看到的，这种方法非常强大。首先，它可以标记单个序列：",ue,G,ce,N,Ce="它还一次处理多个序列，并且API没有任何变化：",re,X,me,_,Ee="它可以根据几个目标进行填充：",Me,x,be,z,Se="它还可以截断序列:",Je,C,ye,E,Re="标记器对象可以处理到特定框架张量的转换，然后可以直接发送到模型。例如，在下面的代码示例中，我们提示标记器从不同的框架返回张量——<code>&quot;pt&quot;</code>返回Py Torch张量，<code>&quot;tf&quot;</code>返回TensorFlow张量，<code>&quot;np&quot;</code>返回NumPy数组：",Te,S,we,R,je,q,qe="如果我们看一下标记器返回的输入 ID，我们会发现它们与之前的略有不同：",de,v,he,Y,fe,Q,ve="一个在开始时添加了一个标记(token) ID，一个在结束时添加了一个标记(token) ID。让我们解码上面的两个ID序列，看看这是怎么回事：",Ie,H,Ue,F,Ze,A,Ye="标记器在开头添加了特殊单词<code>[CLS]</code>，在结尾添加了特殊单词<code>[SEP]</code>。这是因为模型是用这些数据预训练的，所以为了得到相同的推理结果，我们还需要添加它们。请注意，有些模型不添加特殊单词，或者添加不同的单词；模型也可能只在开头或结尾添加这些特殊单词。在任何情况下，标记器都知道需要哪些词符，并将为您处理这些词符。",ge,D,Ve,P,Qe="现在我们已经看到了标记器对象在应用于文本时使用的所有单独步骤，让我们最后一次看看它如何处理多个序列（填充！），非常长的序列（截断！），以及多种类型的张量及其主要API：",ke,I,U,le,K,$e,se,We;Z=new il({props:{fw:d[0]}}),g=new Ne({props:{title:"把它们放在一起",local:"把它们放在一起",headingTag:"h1"}});const He=[rl,cl],L=[];function Fe(e,l){return e[0]==="pt"?0:1}h=Fe(d),f=L[h]=He[h](d),$=new j({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),G=new j({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),X=new j({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),x=new j({props:{code:"JTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtYXhpbXVtJTIwc2VxdWVuY2UlMjBsZW5ndGglMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMmxvbmdlc3QlMjIpJTBBJTBBJTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtb2RlbCUyMG1heCUyMGxlbmd0aCUwQSUyMyUyMCg1MTIlMjBmb3IlMjBCRVJUJTIwb3IlMjBEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiklMEElMEElMjMlMjBXaWxsJTIwcGFkJTIwdGhlJTIwc2VxdWVuY2VzJTIwdXAlMjB0byUyMHRoZSUyMHNwZWNpZmllZCUyMG1heCUyMGxlbmd0aCUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiUyQyUyMG1heF9sZW5ndGglM0Q4KQ==",highlighted:`<span class="hljs-comment"># Will pad the sequences up to the maximum sequence length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the specified max length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),C=new j({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBXaWxsJTIwdHJ1bmNhdGUlMjB0aGUlMjBzZXF1ZW5jZXMlMjB0aGF0JTIwYXJlJTIwbG9uZ2VyJTIwdGhhbiUyMHRoZSUyMG1vZGVsJTIwbWF4JTIwbGVuZ3RoJTBBJTIzJTIwKDUxMiUyMGZvciUyMEJFUlQlMjBvciUyMERpc3RpbEJFUlQpJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQSUwQSUyMyUyMFdpbGwlMjB0cnVuY2F0ZSUyMHRoZSUyMHNlcXVlbmNlcyUyMHRoYXQlMjBhcmUlMjBsb25nZXIlMjB0aGFuJTIwdGhlJTIwc3BlY2lmaWVkJTIwbWF4JTIwbGVuZ3RoJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMG1heF9sZW5ndGglM0Q4JTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Will truncate the sequences that are longer than the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Will truncate the sequences that are longer than the specified max length</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),S=new j({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBSZXR1cm5zJTIwUHlUb3JjaCUyMHRlbnNvcnMlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMFJldHVybnMlMjBUZW5zb3JGbG93JTIwdGVuc29ycyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBJTBBJTIzJTIwUmV0dXJucyUyME51bVB5JTIwYXJyYXlzJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJucCUyMik=",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Returns PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Returns TensorFlow tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Returns NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),R=new Ne({props:{title:"特殊词符(token)",local:"特殊词符(token)",headingTag:"h2"}}),v=new j({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),Y=new j({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),H=new j({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),F=new j({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),D=new Ne({props:{title:"结束：从标记器到模型",local:"结束：从标记器到模型",headingTag:"h2"}});const Ae=[Ml,ml],O=[];function De(e,l){return e[0]==="pt"?0:1}return I=De(d),U=O[I]=Ae[I](d),K=new ul({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-CN/chapter2/6.mdx"}}),{c(){a=J("meta"),i=p(),t=J("p"),T=p(),r(Z.$$.fragment),te=p(),r(g.$$.fragment),ne=p(),f.c(),ee=p(),V=J("p"),V.textContent=Xe,ae=p(),k=J("p"),k.textContent=_e,pe=p(),r($.$$.fragment),oe=p(),W=J("p"),W.innerHTML=xe,ie=p(),B=J("p"),B.textContent=ze,ue=p(),r(G.$$.fragment),ce=p(),N=J("p"),N.textContent=Ce,re=p(),r(X.$$.fragment),me=p(),_=J("p"),_.textContent=Ee,Me=p(),r(x.$$.fragment),be=p(),z=J("p"),z.textContent=Se,Je=p(),r(C.$$.fragment),ye=p(),E=J("p"),E.innerHTML=Re,Te=p(),r(S.$$.fragment),we=p(),r(R.$$.fragment),je=p(),q=J("p"),q.textContent=qe,de=p(),r(v.$$.fragment),he=p(),r(Y.$$.fragment),fe=p(),Q=J("p"),Q.textContent=ve,Ie=p(),r(H.$$.fragment),Ue=p(),r(F.$$.fragment),Ze=p(),A=J("p"),A.innerHTML=Ye,ge=p(),r(D.$$.fragment),Ve=p(),P=J("p"),P.textContent=Qe,ke=p(),U.c(),le=p(),r(K.$$.fragment),$e=p(),se=J("p"),this.h()},l(e){const l=pl("svelte-u9bgzb",document.head);a=y(l,"META",{name:!0,content:!0}),l.forEach(s),i=o(e),t=y(e,"P",{}),Ke(t).forEach(s),T=o(e),m(Z.$$.fragment,e),te=o(e),m(g.$$.fragment,e),ne=o(e),f.l(e),ee=o(e),V=y(e,"P",{"data-svelte-h":!0}),w(V)!=="svelte-9ckwb8"&&(V.textContent=Xe),ae=o(e),k=y(e,"P",{"data-svelte-h":!0}),w(k)!=="svelte-lmb6iw"&&(k.textContent=_e),pe=o(e),m($.$$.fragment,e),oe=o(e),W=y(e,"P",{"data-svelte-h":!0}),w(W)!=="svelte-vr6ivv"&&(W.innerHTML=xe),ie=o(e),B=y(e,"P",{"data-svelte-h":!0}),w(B)!=="svelte-1ys6k4x"&&(B.textContent=ze),ue=o(e),m(G.$$.fragment,e),ce=o(e),N=y(e,"P",{"data-svelte-h":!0}),w(N)!=="svelte-81qtyc"&&(N.textContent=Ce),re=o(e),m(X.$$.fragment,e),me=o(e),_=y(e,"P",{"data-svelte-h":!0}),w(_)!=="svelte-1brznle"&&(_.textContent=Ee),Me=o(e),m(x.$$.fragment,e),be=o(e),z=y(e,"P",{"data-svelte-h":!0}),w(z)!=="svelte-1kgpxe"&&(z.textContent=Se),Je=o(e),m(C.$$.fragment,e),ye=o(e),E=y(e,"P",{"data-svelte-h":!0}),w(E)!=="svelte-k51qrg"&&(E.innerHTML=Re),Te=o(e),m(S.$$.fragment,e),we=o(e),m(R.$$.fragment,e),je=o(e),q=y(e,"P",{"data-svelte-h":!0}),w(q)!=="svelte-1kdyhh7"&&(q.textContent=qe),de=o(e),m(v.$$.fragment,e),he=o(e),m(Y.$$.fragment,e),fe=o(e),Q=y(e,"P",{"data-svelte-h":!0}),w(Q)!=="svelte-10lmw3t"&&(Q.textContent=ve),Ie=o(e),m(H.$$.fragment,e),Ue=o(e),m(F.$$.fragment,e),Ze=o(e),A=y(e,"P",{"data-svelte-h":!0}),w(A)!=="svelte-1g1k4vg"&&(A.innerHTML=Ye),ge=o(e),m(D.$$.fragment,e),Ve=o(e),P=y(e,"P",{"data-svelte-h":!0}),w(P)!=="svelte-su2qfp"&&(P.textContent=Qe),ke=o(e),U.l(e),le=o(e),m(K.$$.fragment,e),$e=o(e),se=y(e,"P",{}),Ke(se).forEach(s),this.h()},h(){Le(a,"name","hf:doc:metadata"),Le(a,"content",Jl)},m(e,l){ol(document.head,a),n(e,i,l),n(e,t,l),n(e,T,l),M(Z,e,l),n(e,te,l),M(g,e,l),n(e,ne,l),L[h].m(e,l),n(e,ee,l),n(e,V,l),n(e,ae,l),n(e,k,l),n(e,pe,l),M($,e,l),n(e,oe,l),n(e,W,l),n(e,ie,l),n(e,B,l),n(e,ue,l),M(G,e,l),n(e,ce,l),n(e,N,l),n(e,re,l),M(X,e,l),n(e,me,l),n(e,_,l),n(e,Me,l),M(x,e,l),n(e,be,l),n(e,z,l),n(e,Je,l),M(C,e,l),n(e,ye,l),n(e,E,l),n(e,Te,l),M(S,e,l),n(e,we,l),M(R,e,l),n(e,je,l),n(e,q,l),n(e,de,l),M(v,e,l),n(e,he,l),M(Y,e,l),n(e,fe,l),n(e,Q,l),n(e,Ie,l),M(H,e,l),n(e,Ue,l),M(F,e,l),n(e,Ze,l),n(e,A,l),n(e,ge,l),M(D,e,l),n(e,Ve,l),n(e,P,l),n(e,ke,l),O[I].m(e,l),n(e,le,l),M(K,e,l),n(e,$e,l),n(e,se,l),We=!0},p(e,[l]){const Pe={};l&1&&(Pe.fw=e[0]),Z.$set(Pe);let Be=h;h=Fe(e),h!==Be&&(el(),u(L[Be],1,1,()=>{L[Be]=null}),Oe(),f=L[h],f||(f=L[h]=He[h](e),f.c()),c(f,1),f.m(ee.parentNode,ee));let Ge=I;I=De(e),I!==Ge&&(el(),u(O[Ge],1,1,()=>{O[Ge]=null}),Oe(),U=O[I],U||(U=O[I]=Ae[I](e),U.c()),c(U,1),U.m(le.parentNode,le))},i(e){We||(c(Z.$$.fragment,e),c(g.$$.fragment,e),c(f),c($.$$.fragment,e),c(G.$$.fragment,e),c(X.$$.fragment,e),c(x.$$.fragment,e),c(C.$$.fragment,e),c(S.$$.fragment,e),c(R.$$.fragment,e),c(v.$$.fragment,e),c(Y.$$.fragment,e),c(H.$$.fragment,e),c(F.$$.fragment,e),c(D.$$.fragment,e),c(U),c(K.$$.fragment,e),We=!0)},o(e){u(Z.$$.fragment,e),u(g.$$.fragment,e),u(f),u($.$$.fragment,e),u(G.$$.fragment,e),u(X.$$.fragment,e),u(x.$$.fragment,e),u(C.$$.fragment,e),u(S.$$.fragment,e),u(R.$$.fragment,e),u(v.$$.fragment,e),u(Y.$$.fragment,e),u(H.$$.fragment,e),u(F.$$.fragment,e),u(D.$$.fragment,e),u(U),u(K.$$.fragment,e),We=!1},d(e){e&&(s(i),s(t),s(T),s(te),s(ne),s(ee),s(V),s(ae),s(k),s(pe),s(oe),s(W),s(ie),s(B),s(ue),s(ce),s(N),s(re),s(me),s(_),s(Me),s(be),s(z),s(Je),s(ye),s(E),s(Te),s(we),s(je),s(q),s(de),s(he),s(fe),s(Q),s(Ie),s(Ue),s(Ze),s(A),s(ge),s(Ve),s(P),s(ke),s(le),s($e),s(se)),s(a),b(Z,e),b(g,e),L[h].d(e),b($,e),b(G,e),b(X,e),b(x,e),b(C,e),b(S,e),b(R,e),b(v,e),b(Y,e),b(H,e),b(F,e),b(D,e),O[I].d(e),b(K,e)}}}const Jl='{"title":"把它们放在一起","local":"把它们放在一起","sections":[{"title":"特殊词符(token)","local":"特殊词符(token)","sections":[],"depth":2},{"title":"结束：从标记器到模型","local":"结束：从标记器到模型","sections":[],"depth":2}],"depth":1}';function yl(d,a,i){let t="pt";return tl(()=>{const T=new URLSearchParams(window.location.search);i(0,t=T.get("fw")||"pt")}),[t]}class Il extends nl{constructor(a){super(),al(this,a,yl,bl,sl,{})}}export{Il as component};
