import{s as sa,f as ea,o as ta,n as ee}from"../chunks/scheduler.37c15a92.js";import{S as aa,i as na,g as d,s as p,r as m,A as Ma,h as j,f as t,c as y,j as ot,u as T,x as b,k as Wl,y as pa,a,v as w,t as i,b as ps,d as c,w as u,e as Kt,p as ys}from"../chunks/index.2bf4358c.js";import{T as rs}from"../chunks/Tip.363c041f.js";import{Y as Ut}from"../chunks/Youtube.1e50a667.js";import{C as h}from"../chunks/CodeBlock.4f5fc1ad.js";import{D as la}from"../chunks/DocNotebookDropdown.d9dfeb24.js";import{F as ya}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as ie,E as ra}from"../chunks/EditOnGithub.8916dc7c.js";function ca(I){let e,r;return e=new la({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section6_tf.ipynb"}]}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function ia(I){let e,r;return e=new la({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section6_pt.ipynb"}]}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function Ja(I){let e,r="预训练语言模型需要一段时间。我们建议您首先通过取消注释以上两行的注释对数据样本运行训练循环，并确保训练成功完成并存储模型。没有什么比最后一步的训练失败更令人沮丧的了，因为你忘记创建一个文件夹或者因为保存路径在训练循环结束时有一个错字！";return{c(){e=d("p"),e.textContent=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-1qfyfm4"&&(e.textContent=r)},m(n,J){a(n,e,J)},p:ee,d(n){n&&t(e)}}}function oa(I){let e,r="✏️ <strong>试试看！</strong> 摆脱所有小于上下文大小的块在这里并不是什么大问题，因为我们使用的是小上下文窗口。随着上下文大小的增加（或者如果您有一个短文档语料库），被丢弃的块的比例也会增加。准备数据的更有效方法是将所有标记化的样本加入一个批次中，每个语料之间有一个<code>eos_token_id</code> 标记, 然后对连接的序列执行分块。作为练习，修改 <code>tokenize()</code>函数以使用该方法。请注意，您需要设置<code>truncation=False</code> 和删除标记生成器中的其他参数以获取完整的标记 ID 序列。";return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-1ph7gud"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},p:ee,d(n){n&&t(e)}}}function Ua(I){let e,r,n,J="通过该配置，我们可以加载新模型。请注意，这是我们刚开始不使用<code>from_pretrained()</code>函数，因为我们实际上是在自己初始化模型：",_,f,G,$,Z;return e=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBURkdQVDJMTUhlYWRNb2RlbCUyQyUyMEF1dG9Db25maWclMEElMEFjb25maWclMjAlM0QlMjBBdXRvQ29uZmlnLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJncHQyJTIyJTJDJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRGxlbih0b2tlbml6ZXIpJTJDJTBBJTIwJTIwJTIwJTIwbl9jdHglM0Rjb250ZXh0X2xlbmd0aCUyQyUwQSUyMCUyMCUyMCUyMGJvc190b2tlbl9pZCUzRHRva2VuaXplci5ib3NfdG9rZW5faWQlMkMlMEElMjAlMjAlMjAlMjBlb3NfdG9rZW5faWQlM0R0b2tlbml6ZXIuZW9zX3Rva2VuX2lkJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFGPT2LMHeadModel, AutoConfig

config = AutoConfig.from_pretrained(
    <span class="hljs-string">&quot;gpt2&quot;</span>,
    vocab_size=<span class="hljs-built_in">len</span>(tokenizer),
    n_ctx=context_length,
    bos_token_id=tokenizer.bos_token_id,
    eos_token_id=tokenizer.eos_token_id,
)`,wrap:!1}}),f=new h({props:{code:"bW9kZWwlMjAlM0QlMjBURkdQVDJMTUhlYWRNb2RlbChjb25maWcpJTBBbW9kZWwobW9kZWwuZHVtbXlfaW5wdXRzKSUyMCUyMCUyMyUyMEJ1aWxkcyUyMHRoZSUyMG1vZGVsJTBBbW9kZWwuc3VtbWFyeSgp",highlighted:`model = TFGPT2LMHeadModel(config)
model(model.dummy_inputs)  <span class="hljs-comment"># Builds the model</span>
model.summary()`,wrap:!1}}),$=new h({props:{code:"X19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18lMEFMYXllciUyMCh0eXBlKSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME91dHB1dCUyMFNoYXBlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwUGFyYW0lMjAlMjMlMjAlMjAlMjAlMEElM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMEF0cmFuc2Zvcm1lciUyMChURkdQVDJNYWluTGF5ZXIlMjBtdWx0aXBsZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDEyNDI0MjQzMiUyMCUwQSUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUwQVRvdGFsJTIwcGFyYW1zJTNBJTIwMTI0JTJDMjQyJTJDNDMyJTBBVHJhaW5hYmxlJTIwcGFyYW1zJTNBJTIwMTI0JTJDMjQyJTJDNDMyJTBBTm9uLXRyYWluYWJsZSUyMHBhcmFtcyUzQSUyMDAlMEFfX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fXw==",highlighted:`_________________________________________________________________
Layer (<span class="hljs-built_in">type</span>)                 Output Shape              Param <span class="hljs-comment">#   </span>
=================================================================
transformer (TFGPT2MainLayer multiple                  <span class="hljs-number">124242432</span> 
=================================================================
Total params: <span class="hljs-number">124</span>,<span class="hljs-number">242</span>,<span class="hljs-number">432</span>
Trainable params: <span class="hljs-number">124</span>,<span class="hljs-number">242</span>,<span class="hljs-number">432</span>
Non-trainable params: <span class="hljs-number">0</span>
_________________________________________________________________`,wrap:!1}}),{c(){m(e.$$.fragment),r=p(),n=d("p"),n.innerHTML=J,_=p(),m(f.$$.fragment),G=p(),m($.$$.fragment)},l(o){T(e.$$.fragment,o),r=y(o),n=j(o,"P",{"data-svelte-h":!0}),b(n)!=="svelte-1ry9t1v"&&(n.innerHTML=J),_=y(o),T(f.$$.fragment,o),G=y(o),T($.$$.fragment,o)},m(o,C){w(e,o,C),a(o,r,C),a(o,n,C),a(o,_,C),w(f,o,C),a(o,G,C),w($,o,C),Z=!0},i(o){Z||(c(e.$$.fragment,o),c(f.$$.fragment,o),c($.$$.fragment,o),Z=!0)},o(o){i(e.$$.fragment,o),i(f.$$.fragment,o),i($.$$.fragment,o),Z=!1},d(o){o&&(t(r),t(n),t(_),t(G)),u(e,o),u(f,o),u($,o)}}}function ma(I){let e,r,n,J="使用该配置，我们可以加载一个新模型。请注意，这是我们第一次不使用 <code>from_pretrained()</code> 函数，因为我们实际上是在自己初始化模型",_,f,G,$,Z;return e=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBHUFQyTE1IZWFkTW9kZWwlMkMlMjBBdXRvQ29uZmlnJTBBJTBBY29uZmlnJTIwJTNEJTIwQXV0b0NvbmZpZy5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyZ3B0MiUyMiUyQyUwQSUyMCUyMCUyMCUyMHZvY2FiX3NpemUlM0RsZW4odG9rZW5pemVyKSUyQyUwQSUyMCUyMCUyMCUyMG5fY3R4JTNEY29udGV4dF9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjBib3NfdG9rZW5faWQlM0R0b2tlbml6ZXIuYm9zX3Rva2VuX2lkJTJDJTBBJTIwJTIwJTIwJTIwZW9zX3Rva2VuX2lkJTNEdG9rZW5pemVyLmVvc190b2tlbl9pZCUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, GPT2LMHeadModel, AutoConfig

config = AutoConfig.from_pretrained(
    <span class="hljs-string">&quot;gpt2&quot;</span>,
    vocab_size=<span class="hljs-built_in">len</span>(tokenizer),
    n_ctx=context_length,
    bos_token_id=tokenizer.bos_token_id,
    eos_token_id=tokenizer.eos_token_id,
)`,wrap:!1}}),f=new h({props:{code:"bW9kZWwlMjAlM0QlMjBHUFQyTE1IZWFkTW9kZWwoY29uZmlnKSUwQW1vZGVsX3NpemUlMjAlM0QlMjBzdW0odC5udW1lbCgpJTIwZm9yJTIwdCUyMGluJTIwbW9kZWwucGFyYW1ldGVycygpKSUwQXByaW50KGYlMjJHUFQtMiUyMHNpemUlM0ElMjAlN0Jtb2RlbF9zaXplJTJGMTAwMCoqMiUzQS4xZiU3RE0lMjBwYXJhbWV0ZXJzJTIyKQ==",highlighted:`model = GPT2LMHeadModel(config)
model_size = <span class="hljs-built_in">sum</span>(t.numel() <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> model.parameters())
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPT-2 size: <span class="hljs-subst">{model_size/<span class="hljs-number">1000</span>**<span class="hljs-number">2</span>:<span class="hljs-number">.1</span>f}</span>M parameters&quot;</span>)`,wrap:!1}}),$=new h({props:{code:"R1BULTIlMjBzaXplJTNBJTIwMTI0LjJNJTIwcGFyYW1ldGVycw==",highlighted:'GPT-<span class="hljs-number">2</span> size: <span class="hljs-number">124.2</span>M parameters',wrap:!1}}),{c(){m(e.$$.fragment),r=p(),n=d("p"),n.innerHTML=J,_=p(),m(f.$$.fragment),G=p(),m($.$$.fragment)},l(o){T(e.$$.fragment,o),r=y(o),n=j(o,"P",{"data-svelte-h":!0}),b(n)!=="svelte-bxdub1"&&(n.innerHTML=J),_=y(o),T(f.$$.fragment,o),G=y(o),T($.$$.fragment,o)},m(o,C){w(e,o,C),a(o,r,C),a(o,n,C),a(o,_,C),w(f,o,C),a(o,G,C),w($,o,C),Z=!0},i(o){Z||(c(e.$$.fragment,o),c(f.$$.fragment,o),c($.$$.fragment,o),Z=!0)},o(o){i(e.$$.fragment,o),i(f.$$.fragment,o),i($.$$.fragment,o),Z=!1},d(o){o&&(t(r),t(n),t(_),t(G)),u(e,o),u(f,o),u($,o)}}}function Ta(I){let e,r;return e=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEF0b2tlbml6ZXIucGFkX3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyLmVvc190b2tlbiUwQWRhdGFfY29sbGF0b3IlMjAlM0QlMjBEYXRhQ29sbGF0b3JGb3JMYW5ndWFnZU1vZGVsaW5nKHRva2VuaXplciUyQyUyMG1sbSUzREZhbHNlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJ0ZiUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function wa(I){let e,r;return e=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEF0b2tlbml6ZXIucGFkX3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyLmVvc190b2tlbiUwQWRhdGFfY29sbGF0b3IlMjAlM0QlMjBEYXRhQ29sbGF0b3JGb3JMYW5ndWFnZU1vZGVsaW5nKHRva2VuaXplciUyQyUyMG1sbSUzREZhbHNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=<span class="hljs-literal">False</span>)`,wrap:!1}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function ua(I){let e,r;return e=new h({props:{code:"aW5wdXRfaWRzJTIwc2hhcGUlM0ElMjAoNSUyQyUyMDEyOCklMEFhdHRlbnRpb25fbWFzayUyMHNoYXBlJTNBJTIwKDUlMkMlMjAxMjgpJTBBbGFiZWxzJTIwc2hhcGUlM0ElMjAoNSUyQyUyMDEyOCk=",highlighted:`input_ids shape: (<span class="hljs-number">5</span>, <span class="hljs-number">128</span>)
attention_mask shape: (<span class="hljs-number">5</span>, <span class="hljs-number">128</span>)
labels shape: (<span class="hljs-number">5</span>, <span class="hljs-number">128</span>)`,wrap:!1}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function da(I){let e,r;return e=new h({props:{code:"aW5wdXRfaWRzJTIwc2hhcGUlM0ElMjB0b3JjaC5TaXplKCU1QjUlMkMlMjAxMjglNUQpJTBBYXR0ZW50aW9uX21hc2slMjBzaGFwZSUzQSUyMHRvcmNoLlNpemUoJTVCNSUyQyUyMDEyOCU1RCklMEFsYWJlbHMlMjBzaGFwZSUzQSUyMHRvcmNoLlNpemUoJTVCNSUyQyUyMDEyOCU1RCk=",highlighted:`input_ids shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>])
attention_mask shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>])
labels shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>])`,wrap:!1}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function Pt(I){let e,r="现在，我们可以使用<code>prepare_tf_dataset()</code>方法，使用上面创建的数据整理器将数据集转换为TensorFlow数据集：",n,J,_;return J=new h({props:{code:"dGZfdHJhaW5fZGF0YXNldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwY29sbGF0ZV9mbiUzRGRhdGFfY29sbGF0b3IlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QzMiUyQyUwQSklMEF0Zl9ldmFsX2RhdGFzZXQlMjAlM0QlMjBtb2RlbC5wcmVwYXJlX3RmX2RhdGFzZXQoJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2RhdGFzZXQlNUIlMjJ2YWxpZCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDMyJTJDJTBBKQ==",highlighted:`tf_train_dataset = model.prepare_tf_dataset(
    tokenized_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">32</span>,
)
tf_eval_dataset = model.prepare_tf_dataset(
    tokenized_dataset[<span class="hljs-string">&quot;valid&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">32</span>,
)`,wrap:!1}}),{c(){e=d("p"),e.innerHTML=r,n=p(),m(J.$$.fragment)},l(f){e=j(f,"P",{"data-svelte-h":!0}),b(e)!=="svelte-1ma9wgc"&&(e.innerHTML=r),n=y(f),T(J.$$.fragment,f)},m(f,G){a(f,e,G),a(f,n,G),w(J,f,G),_=!0},i(f){_||(c(J.$$.fragment,f),_=!0)},o(f){i(J.$$.fragment,f),_=!1},d(f){f&&(t(e),t(n)),u(J,f)}}}function ja(I){let e,r="⚠️  移动输入和标签以对齐它们发生在模型内部，因此数据整理器只需复制输入以创建标签。";return{c(){e=d("p"),e.textContent=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-10szqb"&&(e.textContent=r)},m(n,J){a(n,e,J)},p:ee,d(n){n&&t(e)}}}function ba(I){let e,r="剩下要做的就是配置训练超参数并调用 <code>compile()</code> 和 <code>fit()</code>。我们将使用带有一些预热的学习率调整策略来提高训练的稳定性：",n,J,_,f,G="现在我们可以调用<code>model.fit()，</code>并等待训练完成。你是在完整的训练集还是他的子集上运行，这将分别需要20和2个小时，所以拿一些咖啡和一本好书来阅读！训练完成后，我们可以将模型和分词器推送到中心：",$,Z,o;return J=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEFpbXBvcnQlMjB0ZW5zb3JmbG93JTIwYXMlMjB0ZiUwQSUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMGxlbih0Zl90cmFpbl9kYXRhc2V0KSUwQW9wdGltaXplciUyQyUyMHNjaGVkdWxlJTIwJTNEJTIwY3JlYXRlX29wdGltaXplciglMEElMjAlMjAlMjAlMjBpbml0X2xyJTNENWUtNSUyQyUwQSUyMCUyMCUyMCUyMG51bV93YXJtdXBfc3RlcHMlM0QxXzAwMCUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9zdGVwcyUzRG51bV90cmFpbl9zdGVwcyUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheV9yYXRlJTNEMC4wMSUyQyUwQSklMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplciklMEElMEElMjMlMjBUcmFpbiUyMGluJTIwbWl4ZWQtcHJlY2lzaW9uJTIwZmxvYXQxNiUwQXRmLmtlcmFzLm1peGVkX3ByZWNpc2lvbi5zZXRfZ2xvYmFsX3BvbGljeSglMjJtaXhlZF9mbG9hdDE2JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset)
optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">5e-5</span>,
    num_warmup_steps=<span class="hljs-number">1_000</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)`,wrap:!1}}),Z=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQWNhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2sob3V0cHV0X2RpciUzRCUyMmNvZGVwYXJyb3QtZHMlMjIlMkMlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIpJTBBJTBBbW9kZWwuZml0KHRmX3RyYWluX2RhdGFzZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl9ldmFsX2RhdGFzZXQlMkMlMjBjYWxsYmFja3MlM0QlNUJjYWxsYmFjayU1RCk=",highlighted:`<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

callback = PushToHubCallback(output_dir=<span class="hljs-string">&quot;codeparrot-ds&quot;</span>, tokenizer=tokenizer)

model.fit(tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback])`,wrap:!1}}),{c(){e=d("p"),e.innerHTML=r,n=p(),m(J.$$.fragment),_=p(),f=d("p"),f.innerHTML=G,$=p(),m(Z.$$.fragment)},l(C){e=j(C,"P",{"data-svelte-h":!0}),b(e)!=="svelte-16spe9z"&&(e.innerHTML=r),n=y(C),T(J.$$.fragment,C),_=y(C),f=j(C,"P",{"data-svelte-h":!0}),b(f)!=="svelte-12b6x4w"&&(f.innerHTML=G),$=y(C),T(Z.$$.fragment,C)},m(C,g){a(C,e,g),a(C,n,g),w(J,C,g),a(C,_,g),a(C,f,g),a(C,$,g),w(Z,C,g),o=!0},i(C){o||(c(J.$$.fragment,C),c(Z.$$.fragment,C),o=!0)},o(C){i(J.$$.fragment,C),i(Z.$$.fragment,C),o=!1},d(C){C&&(t(e),t(n),t(_),t(f),t($)),u(J,C),u(Z,C)}}}function fa(I){let e,r="剩下要做的就是配置训练参数并启动 <code>Trainer</code> .我们将使用余弦学习率，并进行一些Warmup和有效批量大小为 256 ( <code>per_device_train_batch_size</code> * <code>gradient_accumulation_steps</code>）。当单个批次不适合内存时使用梯度累积，并通过多次向前/向后传递逐步建立梯度。当我们使用 🤗 Accelerate 创建训练循环时，我们将看到这一点。",n,J,_,f,G="现在我们可以开始 <code>Trainer</code>并等待训练完成。根据您是在整个训练集还是在训练集的一个子集上运行它，这将分别需要 20 或 2 个小时，因此请喝杯咖啡和一本好书来阅读！",$,Z,o,C,g="训练完成后，我们可以将模型和标记器推送到 Hub：",z,V,R;return J=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUwQSUwQWFyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyY29kZXBhcnJvdC1kcyUyMiUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDMyJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0QzMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJzdGVwcyUyMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfc3RlcHMlM0Q1XzAwMCUyQyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0Q1XzAwMCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDglMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMSUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9zdGVwcyUzRDFfMDAwJTJDJTBBJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyX3R5cGUlM0QlMjJjb3NpbmUlMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNENWUtNCUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RlcHMlM0Q1XzAwMCUyQyUwQSUyMCUyMCUyMCUyMGZwMTYlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEYXJncyUyQyUwQSUyMCUyMCUyMCUyMGRhdGFfY29sbGF0b3IlM0RkYXRhX2NvbGxhdG9yJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRHRva2VuaXplZF9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX2RhdGFzZXRzJTVCJTIydmFsaWQlMjIlNUQlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;codeparrot-ds&quot;</span>,
    per_device_train_batch_size=<span class="hljs-number">32</span>,
    per_device_eval_batch_size=<span class="hljs-number">32</span>,
    evaluation_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    eval_steps=<span class="hljs-number">5_000</span>,
    logging_steps=<span class="hljs-number">5_000</span>,
    gradient_accumulation_steps=<span class="hljs-number">8</span>,
    num_train_epochs=<span class="hljs-number">1</span>,
    weight_decay=<span class="hljs-number">0.1</span>,
    warmup_steps=<span class="hljs-number">1_000</span>,
    lr_scheduler_type=<span class="hljs-string">&quot;cosine&quot;</span>,
    learning_rate=<span class="hljs-number">5e-4</span>,
    save_steps=<span class="hljs-number">5_000</span>,
    fp16=<span class="hljs-literal">True</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)

trainer = Trainer(
    model=model,
    tokenizer=tokenizer,
    args=args,
    data_collator=data_collator,
    train_dataset=tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_datasets[<span class="hljs-string">&quot;valid&quot;</span>],
)`,wrap:!1}}),Z=new h({props:{code:"dHJhaW5lci50cmFpbigp",highlighted:"trainer.train()",wrap:!1}}),V=new h({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:"trainer.push_to_hub()",wrap:!1}}),{c(){e=d("p"),e.innerHTML=r,n=p(),m(J.$$.fragment),_=p(),f=d("p"),f.innerHTML=G,$=p(),m(Z.$$.fragment),o=p(),C=d("p"),C.textContent=g,z=p(),m(V.$$.fragment)},l(k){e=j(k,"P",{"data-svelte-h":!0}),b(e)!=="svelte-169ofku"&&(e.innerHTML=r),n=y(k),T(J.$$.fragment,k),_=y(k),f=j(k,"P",{"data-svelte-h":!0}),b(f)!=="svelte-15vmt03"&&(f.innerHTML=G),$=y(k),T(Z.$$.fragment,k),o=y(k),C=j(k,"P",{"data-svelte-h":!0}),b(C)!=="svelte-1ru2fs6"&&(C.textContent=g),z=y(k),T(V.$$.fragment,k)},m(k,B){a(k,e,B),a(k,n,B),w(J,k,B),a(k,_,B),a(k,f,B),a(k,$,B),w(Z,k,B),a(k,o,B),a(k,C,B),a(k,z,B),w(V,k,B),R=!0},i(k){R||(c(J.$$.fragment,k),c(Z.$$.fragment,k),c(V.$$.fragment,k),R=!0)},o(k){i(J.$$.fragment,k),i(Z.$$.fragment,k),i(V.$$.fragment,k),R=!1},d(k){k&&(t(e),t(n),t(_),t(f),t($),t(o),t(C),t(z)),u(J,k),u(Z,k),u(V,k)}}}function Ca(I){let e,r="✏️ <strong>试试看!</strong> 除了<code>TrainingArguments</code> 之外，我们只需要大约30行代码就可以从原始文本到训练GPT-2。 用你自己的数据集试试看，看看你能不能得到好的结果！";return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-8soesa"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},p:ee,d(n){n&&t(e)}}}function ha(I){let e,r='💡 如果您正在使用具有多个 GPU 的计算机，则可以尝试使用 <code>MirroredStrategy</code> 上下文来大幅加快训练速度。您需要创建一个<code>tf.distribute.MirroredStrategy</code>对象，并确保所有的 <code>to_tf_dataset</code> 或 <code>prepare_tf_dataset()</code> 方法以及模型创建和对 <code>fit()</code>的调用都在其 <code>scope()</code> 上下文中运行。您可以查看有关此内容的文档<a href="https://www.tensorflow.org/guide/distributed_training#use_tfdistributestrategy_with_keras_modelfit" rel="nofollow">here</a>.';return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-x27bqq"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},d(n){n&&t(e)}}}function Ia(I){let e,r="💡 如果您可以访问具有多个 GPU 的机器，请尝试在那里运行代码。 <code>Trainer</code>自动管理多台机器，这可以极大地加快训练速度。";return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-tptd03"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},d(n){n&&t(e)}}}function _a(I){let e;function r(_,f){return _[0]==="pt"?Ia:ha}let n=r(I),J=n(I);return{c(){J.c(),e=Kt()},l(_){J.l(_),e=Kt()},m(_,f){J.m(_,f),a(_,e,f)},p(_,f){n!==(n=r(_))&&(J.d(1),J=n(_),J&&(J.c(),J.m(e.parentNode,e)))},d(_){_&&t(e),J.d(_)}}}function ka(I){let e,r;return e=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY291cnNlX21vZGVsJTIwJTNEJTIwVEZHUFQyTE1IZWFkTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmNvZGVwYXJyb3QtZHMlMjIpJTBBY291cnNlX3Rva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmNvZGVwYXJyb3QtZHMlMjIpJTBBcGlwZSUyMCUzRCUyMHBpcGVsaW5lKCUwQSUyMCUyMCUyMCUyMCUyMnRleHQtZ2VuZXJhdGlvbiUyMiUyQyUyMG1vZGVsJTNEY291cnNlX21vZGVsJTJDJTIwdG9rZW5pemVyJTNEY291cnNlX3Rva2VuaXplciUyQyUyMGRldmljZSUzRDAlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

course_model = TFGPT2LMHeadModel.from_pretrained(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds&quot;</span>)
course_tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds&quot;</span>)
pipe = pipeline(
    <span class="hljs-string">&quot;text-generation&quot;</span>, model=course_model, tokenizer=course_tokenizer, device=<span class="hljs-number">0</span>
)`,wrap:!1}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function Za(I){let e,r;return e=new h({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwcGlwZWxpbmUlMEElMEFkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiklMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwdG9yY2guZGV2aWNlKCUyMmNwdSUyMiklMEFwaXBlJTIwJTNEJTIwcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTIwbW9kZWwlM0QlMjJodWdnaW5nZmFjZS1jb3Vyc2UlMkZjb2RlcGFycm90LWRzJTIyJTJDJTIwZGV2aWNlJTNEZGV2aWNlJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
pipe = pipeline(
    <span class="hljs-string">&quot;text-generation&quot;</span>, model=<span class="hljs-string">&quot;huggingface-course/codeparrot-ds&quot;</span>, device=device
)`,wrap:!1}}),{c(){m(e.$$.fragment)},l(n){T(e.$$.fragment,n)},m(n,J){w(e,n,J),r=!0},i(n){r||(c(e.$$.fragment,n),r=!0)},o(n){i(e.$$.fragment,n),r=!1},d(n){u(e,n)}}}function $a(I){let e,r="从这几个例子来看，模型似乎已经学习了 Python 数据科学堆栈的一些语法（当然，在将模型部署到现实世界之前，我们需要对其进行更全面的评估）。然而，有时需要对模型训练进行更多定制才能实现给定用例的必要性能。例如，如果我们想动态更新批量大小或有一个条件训练循环来即时跳过坏示例怎么办？一种选择是将 <code>Trainer</code> 并添加必要的更改，但有时从头开始编写训练循环会更简单。这就是🤗 Accelerate 的用武之地。";return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-fvr59g"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},d(n){n&&t(e)}}}function Ga(I){let e,r="看看这几个例子，似乎模型已经学习了Python数据科学堆栈的一些语法。当然，在将模型部署到现实世界之前，我们需要更彻底地评估模型，但这仍然是一个令人印象深刻的原型。";return{c(){e=d("p"),e.textContent=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-1gy49mm"&&(e.textContent=r)},m(n,J){a(n,e,J)},d(n){n&&t(e)}}}function Ot(I){let e,r,n,J="我们已经看到了如何使用 <code>Trainer</code> ，这可以允许一些自定义。然而，有时我们想要完全控制训练循环，或者我们想要进行一些奇特的更改。在这种情况下 🤗 Accelerate 是一个不错的选择，在本节中，我们将逐步介绍使用它来训练我们的模型的步骤。为了让事情变得更有趣，我们还将在训练循环中添加一些修改。",_,f,G,$,Z="由于我们主要对数据科学库的合理自动填充感兴趣，因此对更多使用这些库的训练样本给予更多权重是有意义的。我们可以通过使用关键字轻松识别这些示例，例如 <code>plt</code>、<code>pd</code>、<code>sk</code>、<code>fit</code>和<code>predict</code>等关键字，我们可以很容易地识别这些示例，这些关键字是matplotlib最常用的导入名称。<code>Pyplot</code>, <code>pandas</code>和<code>sklearn</code>以及后者的拟合/预测模式。如果这些都表示为单个标记，我们可以轻松检查它们是否出现在输入序列中。标记可能有一个空格前缀，因此我们还将在标记器词汇表中检查这些版本。为了验证它是否有效，我们将添加一个测试token ，该token 应拆分为多个tokens：",o,C,g,z,V,R,k="太好了，这似乎很好用！我们现在可以编写一个自定义损失函数，它将输入序列、logits 和我们刚刚选择的关​​键标记作为输入。首先，我们需要对齐 logits 和输入：向右移动一个的输入序列形成标签，因为下一个标记是当前标记的标签。我们可以通过从输入序列的第二个标记开始标记来实现这一点，因为模型无论如何都不会对第一个标记进行预测。然后我们切断最后一个 logit，因为我们没有完整输入序列之后的标记的标签。有了这个，我们可以计算每个样本的损失并计算每个样本中所有关键字的出现次数。最后，我们使用出现次数作为权重计算所有样本的加权平均值。由于我们不想扔掉所有没有关键字的样本，我们将权重加1：",B,A,Vl,v,Je="在我们开始使用这个很棒的新损失函数进行训练之前，我们需要准备一些东西：",El,X,oe="<li>我们需要数据加载器来批量加载数据。</li> <li>我们需要设置权重衰减参数。</li> <li>有时我们想要求值，因此将求值代码包装在一个函数中是有意义的。</li>",zl,N,Ue="让我们从数据加载器开始。我们只需要将数据集的格式设置为 <code>&quot;torch&quot;</code>，然后我们可以将它传递给 PyTorch <code>DataLoader</code> ,同时设置适当的批量大小：",Al,x,vl,F,me="接下来，我们对参数进行分组，以便优化器知道哪些将获得额外的权重衰减。通常，所有偏差和 LayerNorm 权重项都不受此限制；以下我们如何做到这一点：",Nl,Y,cs,nl,Gl="由于我们希望在训练期间定期在验证集上评估模型，因此我们也为此编写一个函数。它只是运行评估数据加载器并收集跨进程的所有损失：",xl,H,is,Ml,gl='通过 <code>evaluate()</code> 函数我们定期可以获取损失值和<a href="/course/chapter7/3">perplexity</a>。接下来，我们重新定义我们的模型以确保我们再次从头开始训练：',Fl,S,Yl,q,Te="然后我们可以定义我们的优化器，使用之前的函数来分割权重衰减的参数：",Hl,D,Sl,L,we="现在让我们准备模型、优化器和数据加载器，以便我们可以开始训练：",ql,K,Dl,P,Ll,O,ue="现在我们已经发送了我们的 <code>train_dataloader</code>到 <code>accelerator.prepare()</code> ，我们可以使用它的长度来计算训练步骤的数量。请记住，我们应该始终在准备好dataloader后执行此操作，因为该方法会改变其长度。我们使用经典线性学习率调度：",Kl,ll,Pl,sl,Js="最后，要将我们的模型推送到 Hub，我们需要创建一个 <code>Repository</code> 工作文件夹中的对象。如果您尚未登录，请先登录 Hugging Face。我们将从我们想要为模型提供的模型 ID 中确定存储库名称（您可以自由地用自己的选择替换 <code>repo_name</code> ；它只需要包含您的用户名，可以使用<code>get_full_repo_name()</code>函数的查看目前的repo_name）：",al,pl,yl,Bl,Ol,el,os="然后我们可以在本地文件夹中克隆该存储库。如果它已经存在，这个本地文件夹应该是我们正在使用的存储库的克隆：",rl,cl,il,Xl,Us="我们现在可以上传我们保存的任何内容 <code>output_dir</code> 通过调用 <code>repo.push_to_hub()</code> 方法。这将帮助我们在每个 epoch 结束时上传中间模型。在我们训练之前，让我们运行一个快速测试，看看评估函数是否正常工作：",Jl,ol,Ul,ml,Tl,Rl,ms="这些损失和困惑度的值非常高，但这并不奇怪，因为我们还没有训练过模型。有了这个，我们已经准备好编写训练脚本的核心部分：训练循环。在训练循环中，我们遍历数据加载器并将批次传递给模型。有了 logits，我们就可以评估我们的自定义损失函数。我们通过梯度累积步骤的数量来缩放损失，以便在聚合更多步骤时不会产生更大的损失。在我们优化之前，我们还剪辑了梯度以获得更好的收敛性。最后，每隔几步，我们就会使用新的 <code>evaluate()</code> 函数评估模型：",wl,Ql,ls,E,de="就是这样 - 您现在拥有自己的因果语言模型（例如 GPT-2）的自定义训练循环，您可以根据自己的需要进一步自定义。",ss,tl,Ts,ul,dl;return e=new ie({props:{title:"用 🤗 Accelerate 训练",local:"用 🤗 Accelerate 训练",headingTag:"h2"}}),f=new Ut({props:{id:"Hm8_PgVTFuc"}}),C=new h({props:{code:"a2V5dG9rZW5faWRzJTIwJTNEJTIwJTVCJTVEJTBBZm9yJTIwa2V5d29yZCUyMGluJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIycGx0JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIycGQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJzayUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmZpdCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnByZWRpY3QlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjIlMjBwbHQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjIlMjBwZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMiUyMHNrJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyJTIwZml0JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyJTIwcHJlZGljdCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnRlc3R0ZXN0JTIyJTJDJTBBJTVEJTNBJTBBJTIwJTIwJTIwJTIwaWRzJTIwJTNEJTIwdG9rZW5pemVyKCU1QmtleXdvcmQlNUQpLmlucHV0X2lkcyU1QjAlNUQlMEElMjAlMjAlMjAlMjBpZiUyMGxlbihpZHMpJTIwJTNEJTNEJTIwMSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtleXRva2VuX2lkcy5hcHBlbmQoaWRzJTVCMCU1RCklMEElMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMktleXdvcmQlMjBoYXMlMjBub3QlMjBzaW5nbGUlMjB0b2tlbiUzQSUyMCU3QmtleXdvcmQlN0QlMjIp",highlighted:`keytoken_ids = []
<span class="hljs-keyword">for</span> keyword <span class="hljs-keyword">in</span> [
    <span class="hljs-string">&quot;plt&quot;</span>,
    <span class="hljs-string">&quot;pd&quot;</span>,
    <span class="hljs-string">&quot;sk&quot;</span>,
    <span class="hljs-string">&quot;fit&quot;</span>,
    <span class="hljs-string">&quot;predict&quot;</span>,
    <span class="hljs-string">&quot; plt&quot;</span>,
    <span class="hljs-string">&quot; pd&quot;</span>,
    <span class="hljs-string">&quot; sk&quot;</span>,
    <span class="hljs-string">&quot; fit&quot;</span>,
    <span class="hljs-string">&quot; predict&quot;</span>,
    <span class="hljs-string">&quot;testtest&quot;</span>,
]:
    ids = tokenizer([keyword]).input_ids[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(ids) == <span class="hljs-number">1</span>:
        keytoken_ids.append(ids[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">else</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Keyword has not single token: <span class="hljs-subst">{keyword}</span>&quot;</span>)`,wrap:!1}}),z=new h({props:{code:"J0tleXdvcmQlMjBoYXMlMjBub3QlMjBzaW5nbGUlMjB0b2tlbiUzQSUyMHRlc3R0ZXN0Jw==",highlighted:'<span class="hljs-string">&#x27;Keyword has not single token: testtest&#x27;</span>',wrap:!1}}),A=new h({props:{code:"ZnJvbSUyMHRvcmNoLm5uJTIwaW1wb3J0JTIwQ3Jvc3NFbnRyb3B5TG9zcyUwQWltcG9ydCUyMHRvcmNoJTBBJTBBJTBBZGVmJTIwa2V5dG9rZW5fd2VpZ2h0ZWRfbG9zcyhpbnB1dHMlMkMlMjBsb2dpdHMlMkMlMjBrZXl0b2tlbl9pZHMlMkMlMjBhbHBoYSUzRDEuMCklM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBTaGlmdCUyMHNvJTIwdGhhdCUyMHRva2VucyUyMCUzQyUyMG4lMjBwcmVkaWN0JTIwbiUwQSUyMCUyMCUyMCUyMHNoaWZ0X2xhYmVscyUyMCUzRCUyMGlucHV0cyU1Qi4uLiUyQyUyMDElM0ElNUQuY29udGlndW91cygpJTBBJTIwJTIwJTIwJTIwc2hpZnRfbG9naXRzJTIwJTNEJTIwbG9naXRzJTVCLi4uJTJDJTIwJTNBLTElMkMlMjAlM0ElNUQuY29udGlndW91cygpJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2FsY3VsYXRlJTIwcGVyLXRva2VuJTIwbG9zcyUwQSUyMCUyMCUyMCUyMGxvc3NfZmN0JTIwJTNEJTIwQ3Jvc3NFbnRyb3B5TG9zcyhyZWR1Y2UlM0RGYWxzZSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mY3Qoc2hpZnRfbG9naXRzLnZpZXcoLTElMkMlMjBzaGlmdF9sb2dpdHMuc2l6ZSgtMSkpJTJDJTIwc2hpZnRfbGFiZWxzLnZpZXcoLTEpKSUwQSUyMCUyMCUyMCUyMCUyMyUyMFJlc2l6ZSUyMGFuZCUyMGF2ZXJhZ2UlMjBsb3NzJTIwcGVyJTIwc2FtcGxlJTBBJTIwJTIwJTIwJTIwbG9zc19wZXJfc2FtcGxlJTIwJTNEJTIwbG9zcy52aWV3KHNoaWZ0X2xvZ2l0cy5zaXplKDApJTJDJTIwc2hpZnRfbG9naXRzLnNpemUoMSkpLm1lYW4oYXhpcyUzRDEpJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2FsY3VsYXRlJTIwYW5kJTIwc2NhbGUlMjB3ZWlnaHRpbmclMEElMjAlMjAlMjAlMjB3ZWlnaHRzJTIwJTNEJTIwdG9yY2guc3RhY2soJTVCKGlucHV0cyUyMCUzRCUzRCUyMGt0KS5mbG9hdCgpJTIwZm9yJTIwa3QlMjBpbiUyMGtleXRva2VuX2lkcyU1RCkuc3VtKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGF4aXMlM0QlNUIwJTJDJTIwMiU1RCUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjB3ZWlnaHRzJTIwJTNEJTIwYWxwaGElMjAqJTIwKDEuMCUyMCUyQiUyMHdlaWdodHMpJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2FsY3VsYXRlJTIwd2VpZ2h0ZWQlMjBhdmVyYWdlJTBBJTIwJTIwJTIwJTIwd2VpZ2h0ZWRfbG9zcyUyMCUzRCUyMChsb3NzX3Blcl9zYW1wbGUlMjAqJTIwd2VpZ2h0cykubWVhbigpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwd2VpZ2h0ZWRfbG9zcw==",highlighted:`<span class="hljs-keyword">from</span> torch.nn <span class="hljs-keyword">import</span> CrossEntropyLoss
<span class="hljs-keyword">import</span> torch


<span class="hljs-keyword">def</span> <span class="hljs-title function_">keytoken_weighted_loss</span>(<span class="hljs-params">inputs, logits, keytoken_ids, alpha=<span class="hljs-number">1.0</span></span>):
    <span class="hljs-comment"># Shift so that tokens &lt; n predict n</span>
    shift_labels = inputs[..., <span class="hljs-number">1</span>:].contiguous()
    shift_logits = logits[..., :-<span class="hljs-number">1</span>, :].contiguous()
    <span class="hljs-comment"># Calculate per-token loss</span>
    loss_fct = CrossEntropyLoss(reduce=<span class="hljs-literal">False</span>)
    loss = loss_fct(shift_logits.view(-<span class="hljs-number">1</span>, shift_logits.size(-<span class="hljs-number">1</span>)), shift_labels.view(-<span class="hljs-number">1</span>))
    <span class="hljs-comment"># Resize and average loss per sample</span>
    loss_per_sample = loss.view(shift_logits.size(<span class="hljs-number">0</span>), shift_logits.size(<span class="hljs-number">1</span>)).mean(axis=<span class="hljs-number">1</span>)
    <span class="hljs-comment"># Calculate and scale weighting</span>
    weights = torch.stack([(inputs == kt).<span class="hljs-built_in">float</span>() <span class="hljs-keyword">for</span> kt <span class="hljs-keyword">in</span> keytoken_ids]).<span class="hljs-built_in">sum</span>(
        axis=[<span class="hljs-number">0</span>, <span class="hljs-number">2</span>]
    )
    weights = alpha * (<span class="hljs-number">1.0</span> + weights)
    <span class="hljs-comment"># Calculate weighted average</span>
    weighted_loss = (loss_per_sample * weights).mean()
    <span class="hljs-keyword">return</span> weighted_loss`,wrap:!1}}),x=new h({props:{code:"ZnJvbSUyMHRvcmNoLnV0aWxzLmRhdGEuZGF0YWxvYWRlciUyMGltcG9ydCUyMERhdGFMb2FkZXIlMEElMEF0b2tlbml6ZWRfZGF0YXNldC5zZXRfZm9ybWF0KCUyMnRvcmNoJTIyKSUwQXRyYWluX2RhdGFsb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKHRva2VuaXplZF9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQlMkMlMjBiYXRjaF9zaXplJTNEMzIlMkMlMjBzaHVmZmxlJTNEVHJ1ZSklMEFldmFsX2RhdGFsb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKHRva2VuaXplZF9kYXRhc2V0JTVCJTIydmFsaWQlMjIlNUQlMkMlMjBiYXRjaF9zaXplJTNEMzIp",highlighted:`<span class="hljs-keyword">from</span> torch.utils.data.dataloader <span class="hljs-keyword">import</span> DataLoader

tokenized_dataset.set_format(<span class="hljs-string">&quot;torch&quot;</span>)
train_dataloader = DataLoader(tokenized_dataset[<span class="hljs-string">&quot;train&quot;</span>], batch_size=<span class="hljs-number">32</span>, shuffle=<span class="hljs-literal">True</span>)
eval_dataloader = DataLoader(tokenized_dataset[<span class="hljs-string">&quot;valid&quot;</span>], batch_size=<span class="hljs-number">32</span>)`,wrap:!1}}),Y=new h({props:{code:"d2VpZ2h0X2RlY2F5JTIwJTNEJTIwMC4xJTBBJTBBJTBBZGVmJTIwZ2V0X2dyb3VwZWRfcGFyYW1zKG1vZGVsJTJDJTIwbm9fZGVjYXklM0QlNUIlMjJiaWFzJTIyJTJDJTIwJTIyTGF5ZXJOb3JtLndlaWdodCUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjBwYXJhbXNfd2l0aF93ZCUyQyUyMHBhcmFtc193aXRob3V0X3dkJTIwJTNEJTIwJTVCJTVEJTJDJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwbiUyQyUyMHAlMjBpbiUyMG1vZGVsLm5hbWVkX3BhcmFtZXRlcnMoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwYW55KG5kJTIwaW4lMjBuJTIwZm9yJTIwbmQlMjBpbiUyMG5vX2RlY2F5KSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhcmFtc193aXRob3V0X3dkLmFwcGVuZChwKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwYXJhbXNfd2l0aF93ZC5hcHBlbmQocCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJwYXJhbXMlMjIlM0ElMjBwYXJhbXNfd2l0aF93ZCUyQyUyMCUyMndlaWdodF9kZWNheSUyMiUzQSUyMHdlaWdodF9kZWNheSU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnBhcmFtcyUyMiUzQSUyMHBhcmFtc193aXRob3V0X3dkJTJDJTIwJTIyd2VpZ2h0X2RlY2F5JTIyJTNBJTIwMC4wJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTVE",highlighted:`weight_decay = <span class="hljs-number">0.1</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_grouped_params</span>(<span class="hljs-params">model, no_decay=[<span class="hljs-string">&quot;bias&quot;</span>, <span class="hljs-string">&quot;LayerNorm.weight&quot;</span>]</span>):
    params_with_wd, params_without_wd = [], []
    <span class="hljs-keyword">for</span> n, p <span class="hljs-keyword">in</span> model.named_parameters():
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">any</span>(nd <span class="hljs-keyword">in</span> n <span class="hljs-keyword">for</span> nd <span class="hljs-keyword">in</span> no_decay):
            params_without_wd.append(p)
        <span class="hljs-keyword">else</span>:
            params_with_wd.append(p)
    <span class="hljs-keyword">return</span> [
        {<span class="hljs-string">&quot;params&quot;</span>: params_with_wd, <span class="hljs-string">&quot;weight_decay&quot;</span>: weight_decay},
        {<span class="hljs-string">&quot;params&quot;</span>: params_without_wd, <span class="hljs-string">&quot;weight_decay&quot;</span>: <span class="hljs-number">0.0</span>},
    ]`,wrap:!1}}),H=new h({props:{code:"ZGVmJTIwZXZhbHVhdGUoKSUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLmV2YWwoKSUwQSUyMCUyMCUyMCUyMGxvc3NlcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMHN0ZXAlMkMlMjBiYXRjaCUyMGluJTIwZW51bWVyYXRlKGV2YWxfZGF0YWxvYWRlciklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTJDJTIwbGFiZWxzJTNEYmF0Y2glNUIlMjJpbnB1dF9pZHMlMjIlNUQpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9zc2VzLmFwcGVuZChhY2NlbGVyYXRvci5nYXRoZXIob3V0cHV0cy5sb3NzKSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwdG9yY2gubWVhbih0b3JjaC5jYXQobG9zc2VzKSklMEElMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwZXJwbGV4aXR5JTIwJTNEJTIwdG9yY2guZXhwKGxvc3MpJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwT3ZlcmZsb3dFcnJvciUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBlcnBsZXhpdHklMjAlM0QlMjBmbG9hdCglMjJpbmYlMjIpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwbG9zcy5pdGVtKCklMkMlMjBwZXJwbGV4aXR5Lml0ZW0oKQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">evaluate</span>():
    model.<span class="hljs-built_in">eval</span>()
    losses = []
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            outputs = model(batch[<span class="hljs-string">&quot;input_ids&quot;</span>], labels=batch[<span class="hljs-string">&quot;input_ids&quot;</span>])

        losses.append(accelerator.gather(outputs.loss))
    loss = torch.mean(torch.cat(losses))
    <span class="hljs-keyword">try</span>:
        perplexity = torch.exp(loss)
    <span class="hljs-keyword">except</span> OverflowError:
        perplexity = <span class="hljs-built_in">float</span>(<span class="hljs-string">&quot;inf&quot;</span>)
    <span class="hljs-keyword">return</span> loss.item(), perplexity.item()`,wrap:!1}}),S=new h({props:{code:"bW9kZWwlMjAlM0QlMjBHUFQyTE1IZWFkTW9kZWwoY29uZmlnKQ==",highlighted:"model = GPT2LMHeadModel(config)",wrap:!1}}),D=new h({props:{code:"ZnJvbSUyMHRvcmNoLm9wdGltJTIwaW1wb3J0JTIwQWRhbVclMEElMEFvcHRpbWl6ZXIlMjAlM0QlMjBBZGFtVyhnZXRfZ3JvdXBlZF9wYXJhbXMobW9kZWwpJTJDJTIwbHIlM0Q1ZS00KQ==",highlighted:`<span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(get_grouped_params(model), lr=<span class="hljs-number">5e-4</span>)`,wrap:!1}}),K=new h({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZnAxNiUzRFRydWUpJTBBJTBBbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTJDJTIwZXZhbF9kYXRhbG9hZGVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZSglMEElMjAlMjAlMjAlMjBtb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMHRyYWluX2RhdGFsb2FkZXIlMkMlMjBldmFsX2RhdGFsb2FkZXIlMEEp",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator(fp16=<span class="hljs-literal">True</span>)

model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,wrap:!1}}),P=new rs({props:{$$slots:{default:[ga]},$$scope:{ctx:I}}}),ll=new h({props:{code:"bnVtX3RyYWluX2Vwb2NocyUyMCUzRCUyMDElMEFudW1fdXBkYXRlX3N0ZXBzX3Blcl9lcG9jaCUyMCUzRCUyMGxlbih0cmFpbl9kYXRhbG9hZGVyKSUwQW51bV90cmFpbmluZ19zdGVwcyUyMCUzRCUyMG51bV90cmFpbl9lcG9jaHMlMjAqJTIwbnVtX3VwZGF0ZV9zdGVwc19wZXJfZXBvY2glMEElMEFscl9zY2hlZHVsZXIlMjAlM0QlMjBnZXRfc2NoZWR1bGVyKCUwQSUyMCUyMCUyMCUyMG5hbWUlM0QlMjJsaW5lYXIlMjIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIlM0RvcHRpbWl6ZXIlMkMlMEElMjAlMjAlMjAlMjBudW1fd2FybXVwX3N0ZXBzJTNEMV8wMDAlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5pbmdfc3RlcHMlM0RudW1fdHJhaW5pbmdfc3RlcHMlMkMlMEEp",highlighted:`num_train_epochs = <span class="hljs-number">1</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    name=<span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">1_000</span>,
    num_training_steps=num_training_steps,
)`,wrap:!1}}),pl=new h({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMFJlcG9zaXRvcnklMkMlMjBnZXRfZnVsbF9yZXBvX25hbWUlMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyY29kZXBhcnJvdC1kcy1hY2NlbGVyYXRlJTIyJTBBcmVwb19uYW1lJTIwJTNEJTIwZ2V0X2Z1bGxfcmVwb19uYW1lKG1vZGVsX25hbWUpJTBBcmVwb19uYW1l",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository, get_full_repo_name

model_name = <span class="hljs-string">&quot;codeparrot-ds-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`,wrap:!1}}),Bl=new h({props:{code:"J3NndWdnZXIlMkZjb2RlcGFycm90LWRzLWFjY2VsZXJhdGUn",highlighted:'<span class="hljs-string">&#x27;sgugger/codeparrot-ds-accelerate&#x27;</span>',wrap:!1}}),cl=new h({props:{code:"b3V0cHV0X2RpciUyMCUzRCUyMCUyMmNvZGVwYXJyb3QtZHMtYWNjZWxlcmF0ZSUyMiUwQXJlcG8lMjAlM0QlMjBSZXBvc2l0b3J5KG91dHB1dF9kaXIlMkMlMjBjbG9uZV9mcm9tJTNEcmVwb19uYW1lKQ==",highlighted:`output_dir = <span class="hljs-string">&quot;codeparrot-ds-accelerate&quot;</span>
repo = Repository(output_dir, clone_from=repo_name)`,wrap:!1}}),ol=new h({props:{code:"ZXZhbHVhdGUoKQ==",highlighted:"evaluate()",wrap:!1}}),ml=new h({props:{code:"KDEwLjkzNDEyNjg1Mzk0Mjg3MSUyQyUyMDU2MDU3LjE0NDUzMTI1KQ==",highlighted:'(<span class="hljs-number">10.934126853942871</span>, <span class="hljs-number">56057.14453125</span>)',wrap:!1}}),Ql=new h({props:{code:"ZnJvbSUyMHRxZG0ubm90ZWJvb2slMjBpbXBvcnQlMjB0cWRtJTBBJTBBZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIwJTNEJTIwOCUwQWV2YWxfc3RlcHMlMjAlM0QlMjA1XzAwMCUwQSUwQW1vZGVsLnRyYWluKCklMEFjb21wbGV0ZWRfc3RlcHMlMjAlM0QlMjAwJTBBZm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV90cmFpbl9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwc3RlcCUyQyUyMGJhdGNoJTIwaW4lMjB0cWRtKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVudW1lcmF0ZSh0cmFpbl9kYXRhbG9hZGVyJTJDJTIwc3RhcnQlM0QxKSUyQyUyMHRvdGFsJTNEbnVtX3RyYWluaW5nX3N0ZXBzJTBBJTIwJTIwJTIwJTIwKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvZ2l0cyUyMCUzRCUyMG1vZGVsKGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEKS5sb2dpdHMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwa2V5dG9rZW5fd2VpZ2h0ZWRfbG9zcyhiYXRjaCU1QiUyMmlucHV0X2lkcyUyMiU1RCUyQyUyMGxvZ2l0cyUyQyUyMGtleXRva2VuX2lkcyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHN0ZXAlMjAlMjUlMjAxMDAlMjAlM0QlM0QlMjAwJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IucHJpbnQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybHIlMjIlM0ElMjBnZXRfbHIoKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnNhbXBsZXMlMjIlM0ElMjBzdGVwJTIwKiUyMHNhbXBsZXNfcGVyX3N0ZXAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGVwcyUyMiUzQSUyMGNvbXBsZXRlZF9zdGVwcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxvc3MlMkZ0cmFpbiUyMiUzQSUyMGxvc3MuaXRlbSgpJTIwKiUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zcyUyMCUyRiUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBzdGVwJTIwJTI1JTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIwJTNEJTNEJTIwMCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmNsaXBfZ3JhZF9ub3JtXyhtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjAxLjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxyX3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb21wbGV0ZWRfc3RlcHMlMjAlMkIlM0QlMjAxJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjAoc3RlcCUyMCUyNSUyMChldmFsX3N0ZXBzJTIwKiUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcykpJTIwJTNEJTNEJTIwMCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV2YWxfbG9zcyUyQyUyMHBlcnBsZXhpdHklMjAlM0QlMjBldmFsdWF0ZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IucHJpbnQoJTdCJTIybG9zcyUyRmV2YWwlMjIlM0ElMjBldmFsX2xvc3MlMkMlMjAlMjJwZXJwbGV4aXR5JTIyJTNBJTIwcGVycGxleGl0eSU3RCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbC50cmFpbigpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3Iud2FpdF9mb3JfZXZlcnlvbmUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHVud3JhcHBlZF9tb2RlbCUyMCUzRCUyMGFjY2VsZXJhdG9yLnVud3JhcF9tb2RlbChtb2RlbCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB1bndyYXBwZWRfbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKG91dHB1dF9kaXIlMkMlMjBzYXZlX2Z1bmN0aW9uJTNEYWNjZWxlcmF0b3Iuc2F2ZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGFjY2VsZXJhdG9yLmlzX21haW5fcHJvY2VzcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRva2VuaXplci5zYXZlX3ByZXRyYWluZWQob3V0cHV0X2RpciklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXBvLnB1c2hfdG9faHViKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbW1pdF9tZXNzYWdlJTNEZiUyMlRyYWluaW5nJTIwaW4lMjBwcm9ncmVzcyUyMHN0ZXAlMjAlN0JzdGVwJTdEJTIyJTJDJTIwYmxvY2tpbmclM0RGYWxzZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCk=",highlighted:`<span class="hljs-keyword">from</span> tqdm.notebook <span class="hljs-keyword">import</span> tqdm

gradient_accumulation_steps = <span class="hljs-number">8</span>
eval_steps = <span class="hljs-number">5_000</span>

model.train()
completed_steps = <span class="hljs-number">0</span>
<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> tqdm(
        <span class="hljs-built_in">enumerate</span>(train_dataloader, start=<span class="hljs-number">1</span>), total=num_training_steps
    ):
        logits = model(batch[<span class="hljs-string">&quot;input_ids&quot;</span>]).logits
        loss = keytoken_weighted_loss(batch[<span class="hljs-string">&quot;input_ids&quot;</span>], logits, keytoken_ids)
        <span class="hljs-keyword">if</span> step % <span class="hljs-number">100</span> == <span class="hljs-number">0</span>:
            accelerator.<span class="hljs-built_in">print</span>(
                {
                    <span class="hljs-string">&quot;lr&quot;</span>: get_lr(),
                    <span class="hljs-string">&quot;samples&quot;</span>: step * samples_per_step,
                    <span class="hljs-string">&quot;steps&quot;</span>: completed_steps,
                    <span class="hljs-string">&quot;loss/train&quot;</span>: loss.item() * gradient_accumulation_steps,
                }
            )
        loss = loss / gradient_accumulation_steps
        accelerator.backward(loss)
        <span class="hljs-keyword">if</span> step % gradient_accumulation_steps == <span class="hljs-number">0</span>:
            accelerator.clip_grad_norm_(model.parameters(), <span class="hljs-number">1.0</span>)
            optimizer.step()
            lr_scheduler.step()
            optimizer.zero_grad()
            completed_steps += <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> (step % (eval_steps * gradient_accumulation_steps)) == <span class="hljs-number">0</span>:
            eval_loss, perplexity = evaluate()
            accelerator.<span class="hljs-built_in">print</span>({<span class="hljs-string">&quot;loss/eval&quot;</span>: eval_loss, <span class="hljs-string">&quot;perplexity&quot;</span>: perplexity})
            model.train()
            accelerator.wait_for_everyone()
            unwrapped_model = accelerator.unwrap_model(model)
            unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
            <span class="hljs-keyword">if</span> accelerator.is_main_process:
                tokenizer.save_pretrained(output_dir)
                repo.push_to_hub(
                    commit_message=<span class="hljs-string">f&quot;Training in progress step <span class="hljs-subst">{step}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
                )`,wrap:!1}}),tl=new rs({props:{$$slots:{default:[Ba]},$$scope:{ctx:I}}}),ul=new rs({props:{$$slots:{default:[Xa]},$$scope:{ctx:I}}}),{c(){m(e.$$.fragment),r=p(),n=d("p"),n.innerHTML=J,_=p(),m(f.$$.fragment),G=p(),$=d("p"),$.innerHTML=Z,o=p(),m(C.$$.fragment),g=p(),m(z.$$.fragment),V=p(),R=d("p"),R.textContent=k,B=p(),m(A.$$.fragment),Vl=p(),v=d("p"),v.textContent=Je,El=p(),X=d("ul"),X.innerHTML=oe,zl=p(),N=d("p"),N.innerHTML=Ue,Al=p(),m(x.$$.fragment),vl=p(),F=d("p"),F.textContent=me,Nl=p(),m(Y.$$.fragment),cs=p(),nl=d("p"),nl.textContent=Gl,xl=p(),m(H.$$.fragment),is=p(),Ml=d("p"),Ml.innerHTML=gl,Fl=p(),m(S.$$.fragment),Yl=p(),q=d("p"),q.textContent=Te,Hl=p(),m(D.$$.fragment),Sl=p(),L=d("p"),L.textContent=we,ql=p(),m(K.$$.fragment),Dl=p(),m(P.$$.fragment),Ll=p(),O=d("p"),O.innerHTML=ue,Kl=p(),m(ll.$$.fragment),Pl=p(),sl=d("p"),sl.innerHTML=Js,al=p(),m(pl.$$.fragment),yl=p(),m(Bl.$$.fragment),Ol=p(),el=d("p"),el.textContent=os,rl=p(),m(cl.$$.fragment),il=p(),Xl=d("p"),Xl.innerHTML=Us,Jl=p(),m(ol.$$.fragment),Ul=p(),m(ml.$$.fragment),Tl=p(),Rl=d("p"),Rl.innerHTML=ms,wl=p(),m(Ql.$$.fragment),ls=p(),E=d("p"),E.textContent=de,ss=p(),m(tl.$$.fragment),Ts=p(),m(ul.$$.fragment)},l(s){T(e.$$.fragment,s),r=y(s),n=j(s,"P",{"data-svelte-h":!0}),b(n)!=="svelte-wo5ise"&&(n.innerHTML=J),_=y(s),T(f.$$.fragment,s),G=y(s),$=j(s,"P",{"data-svelte-h":!0}),b($)!=="svelte-1nwmwek"&&($.innerHTML=Z),o=y(s),T(C.$$.fragment,s),g=y(s),T(z.$$.fragment,s),V=y(s),R=j(s,"P",{"data-svelte-h":!0}),b(R)!=="svelte-1q9knf9"&&(R.textContent=k),B=y(s),T(A.$$.fragment,s),Vl=y(s),v=j(s,"P",{"data-svelte-h":!0}),b(v)!=="svelte-11s0wad"&&(v.textContent=Je),El=y(s),X=j(s,"UL",{"data-svelte-h":!0}),b(X)!=="svelte-1i4kynx"&&(X.innerHTML=oe),zl=y(s),N=j(s,"P",{"data-svelte-h":!0}),b(N)!=="svelte-16orru1"&&(N.innerHTML=Ue),Al=y(s),T(x.$$.fragment,s),vl=y(s),F=j(s,"P",{"data-svelte-h":!0}),b(F)!=="svelte-gp372v"&&(F.textContent=me),Nl=y(s),T(Y.$$.fragment,s),cs=y(s),nl=j(s,"P",{"data-svelte-h":!0}),b(nl)!=="svelte-l2pfqq"&&(nl.textContent=Gl),xl=y(s),T(H.$$.fragment,s),is=y(s),Ml=j(s,"P",{"data-svelte-h":!0}),b(Ml)!=="svelte-obgqhd"&&(Ml.innerHTML=gl),Fl=y(s),T(S.$$.fragment,s),Yl=y(s),q=j(s,"P",{"data-svelte-h":!0}),b(q)!=="svelte-55938r"&&(q.textContent=Te),Hl=y(s),T(D.$$.fragment,s),Sl=y(s),L=j(s,"P",{"data-svelte-h":!0}),b(L)!=="svelte-jw4nrn"&&(L.textContent=we),ql=y(s),T(K.$$.fragment,s),Dl=y(s),T(P.$$.fragment,s),Ll=y(s),O=j(s,"P",{"data-svelte-h":!0}),b(O)!=="svelte-s9h0n3"&&(O.innerHTML=ue),Kl=y(s),T(ll.$$.fragment,s),Pl=y(s),sl=j(s,"P",{"data-svelte-h":!0}),b(sl)!=="svelte-wqdk2u"&&(sl.innerHTML=Js),al=y(s),T(pl.$$.fragment,s),yl=y(s),T(Bl.$$.fragment,s),Ol=y(s),el=j(s,"P",{"data-svelte-h":!0}),b(el)!=="svelte-tsfefx"&&(el.textContent=os),rl=y(s),T(cl.$$.fragment,s),il=y(s),Xl=j(s,"P",{"data-svelte-h":!0}),b(Xl)!=="svelte-1fn02u2"&&(Xl.innerHTML=Us),Jl=y(s),T(ol.$$.fragment,s),Ul=y(s),T(ml.$$.fragment,s),Tl=y(s),Rl=j(s,"P",{"data-svelte-h":!0}),b(Rl)!=="svelte-yuwx8a"&&(Rl.innerHTML=ms),wl=y(s),T(Ql.$$.fragment,s),ls=y(s),E=j(s,"P",{"data-svelte-h":!0}),b(E)!=="svelte-6gpatt"&&(E.textContent=de),ss=y(s),T(tl.$$.fragment,s),Ts=y(s),T(ul.$$.fragment,s)},m(s,U){w(e,s,U),a(s,r,U),a(s,n,U),a(s,_,U),w(f,s,U),a(s,G,U),a(s,$,U),a(s,o,U),w(C,s,U),a(s,g,U),w(z,s,U),a(s,V,U),a(s,R,U),a(s,B,U),w(A,s,U),a(s,Vl,U),a(s,v,U),a(s,El,U),a(s,X,U),a(s,zl,U),a(s,N,U),a(s,Al,U),w(x,s,U),a(s,vl,U),a(s,F,U),a(s,Nl,U),w(Y,s,U),a(s,cs,U),a(s,nl,U),a(s,xl,U),w(H,s,U),a(s,is,U),a(s,Ml,U),a(s,Fl,U),w(S,s,U),a(s,Yl,U),a(s,q,U),a(s,Hl,U),w(D,s,U),a(s,Sl,U),a(s,L,U),a(s,ql,U),w(K,s,U),a(s,Dl,U),w(P,s,U),a(s,Ll,U),a(s,O,U),a(s,Kl,U),w(ll,s,U),a(s,Pl,U),a(s,sl,U),a(s,al,U),w(pl,s,U),a(s,yl,U),w(Bl,s,U),a(s,Ol,U),a(s,el,U),a(s,rl,U),w(cl,s,U),a(s,il,U),a(s,Xl,U),a(s,Jl,U),w(ol,s,U),a(s,Ul,U),w(ml,s,U),a(s,Tl,U),a(s,Rl,U),a(s,wl,U),w(Ql,s,U),a(s,ls,U),a(s,E,U),a(s,ss,U),w(tl,s,U),a(s,Ts,U),w(ul,s,U),dl=!0},i(s){dl||(c(e.$$.fragment,s),c(f.$$.fragment,s),c(C.$$.fragment,s),c(z.$$.fragment,s),c(A.$$.fragment,s),c(x.$$.fragment,s),c(Y.$$.fragment,s),c(H.$$.fragment,s),c(S.$$.fragment,s),c(D.$$.fragment,s),c(K.$$.fragment,s),c(P.$$.fragment,s),c(ll.$$.fragment,s),c(pl.$$.fragment,s),c(Bl.$$.fragment,s),c(cl.$$.fragment,s),c(ol.$$.fragment,s),c(ml.$$.fragment,s),c(Ql.$$.fragment,s),c(tl.$$.fragment,s),c(ul.$$.fragment,s),dl=!0)},o(s){i(e.$$.fragment,s),i(f.$$.fragment,s),i(C.$$.fragment,s),i(z.$$.fragment,s),i(A.$$.fragment,s),i(x.$$.fragment,s),i(Y.$$.fragment,s),i(H.$$.fragment,s),i(S.$$.fragment,s),i(D.$$.fragment,s),i(K.$$.fragment,s),i(P.$$.fragment,s),i(ll.$$.fragment,s),i(pl.$$.fragment,s),i(Bl.$$.fragment,s),i(cl.$$.fragment,s),i(ol.$$.fragment,s),i(ml.$$.fragment,s),i(Ql.$$.fragment,s),i(tl.$$.fragment,s),i(ul.$$.fragment,s),dl=!1},d(s){s&&(t(r),t(n),t(_),t(G),t($),t(o),t(g),t(V),t(R),t(B),t(Vl),t(v),t(El),t(X),t(zl),t(N),t(Al),t(vl),t(F),t(Nl),t(cs),t(nl),t(xl),t(is),t(Ml),t(Fl),t(Yl),t(q),t(Hl),t(Sl),t(L),t(ql),t(Dl),t(Ll),t(O),t(Kl),t(Pl),t(sl),t(al),t(yl),t(Ol),t(el),t(rl),t(il),t(Xl),t(Jl),t(Ul),t(Tl),t(Rl),t(wl),t(ls),t(E),t(ss),t(Ts)),u(e,s),u(f,s),u(C,s),u(z,s),u(A,s),u(x,s),u(Y,s),u(H,s),u(S,s),u(D,s),u(K,s),u(P,s),u(ll,s),u(pl,s),u(Bl,s),u(cl,s),u(ol,s),u(ml,s),u(Ql,s),u(tl,s),u(ul,s)}}}function ga(I){let e,r='🚨 如果您在 TPU 上进行训练，则需要将从上面的单元格开始的所有代码移动到专用的训练函数中。有关详细信息，请参阅 <a href="/course/chapter3">第 3 章</a> for more details.';return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-875tum"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},p:ee,d(n){n&&t(e)}}}function Ba(I){let e,r="✏️ <strong>试试看!</strong> 创建适合您的用例的自定义损失函数，或在训练循环中添加另一个自定义步骤。";return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-12rhmpo"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},p:ee,d(n){n&&t(e)}}}function Xa(I){let e,r="✏️ <strong>试试看!</strong> 在运行长时间的训练实验时，最好使用 TensorBoard 或 Weights Biases 等工具记录重要指标。向训练循环添加适当的日志记录，以便您始终可以检查训练的进行情况。going.";return{c(){e=d("p"),e.innerHTML=r},l(n){e=j(n,"P",{"data-svelte-h":!0}),b(e)!=="svelte-nn41xh"&&(e.innerHTML=r)},m(n,J){a(n,e,J)},p:ee,d(n){n&&t(e)}}}function Ra(I){let e,r,n,J,_,f,G,$,Z,o,C,g,z='到目前为止，我们主要使用预训练模型，并通过重用预训练的权重来针对新用例对它们进行微调。正如我们在<a href="/course/chapter1">第一章</a>, 这通常称为迁移学习，这是将 Transformer 模型应用于大多数标记数据稀疏的现实世界用例的非常成功的策略。在本章中，我们将采用不同的方法并从头开始训练一个全新的模型。如果您有大量数据并且它与用于可用模型的预训练数据有很大不同，那么这是一个很好的方法。然而，它也需要更多的计算资源来预训练语言模型，而不仅仅是微调现有的模型。训练新模型有意义的示例包括由音符、分子序列（如 DNA）或编程语言组成的数据集。后者最近受到关注，这要归功于 TabNine 和 GitHub 的 Copilot 等工具，它们由 OpenAI 的 Codex 模型提供支持，可以生成长代码序列。这种文本生成任务最好使用自回归或因果语言模型（例如 GPT-2）来解决。',V,R,k="在本节中，我们将构建代码生成模型的缩小版本：我们将使用 Python 代码的子集专注于单行完成而不是完整的函数或类。在 Python 中处理数据时，您会经常接触 Python 数据科学堆栈，包括 <code>matplotlib</code> , <code>seaborn</code> , <code>pandas</code> ， 和 <code>scikit-learn</code> 库。在使用这些框架时，通常需要查找特定的命令，因此如果我们可以使用模型来为我们完成这些调用，那就太好了。",B,A,Vl,v,Je='在<a href="/course/chapter6">第六章</a> 我们创建了一个高效的分词器来处理 Python 源代码，但我们仍然需要一个大规模数据集来预训练模型。在这里，我们将我们的分词器应用到源自 GitHub 存储库的 Python 代码语料库。然后我们将使用 <code>Trainer</code> API 和 🤗 Accelerate 来训练模型。让我们开始吧！',El,X,oe,zl,N,Ue='这实际上展示了使用本节中训练并上传到 Hub 的模型。你可以在<a href="https://huggingface.co/huggingface-course/codeparrot-ds?text=plt.imshow%28" rel="nofollow">这里</a>找到。请注意，由于在文本生成过程中发生了一些随机化，您可能会得到略有不同的结果。',Al,x,vl,F,me='Python 代码可以从 GitHub 等代码存储库中获得，我们可以通过抓取每个 Python 存储库来使用它们来创建数据集。这是在<a href="https://learning.oreilly.com/library/view/natural-language-processing/9781098136789/" rel="nofollow">Transformers textbook</a>预训练大型的GPT-2 模型。使用大约 180 GB 的 GitHub 转储，其中包含大约 2000 万个 Python 文件，称为 <code>codeparrot</code> ，作者构建了一个数据集，然后在<a href="https://huggingface.co/datasets/transformersbook/codeparrot" rel="nofollow">Hugging Face Hub</a>上分享出来了.',Nl,Y,cs="然而，对完整语料库的训练既耗时又费力，我们只需要与 Python 数据科学堆栈相关的数据集子集。所以，让我们开始过滤 <code>codeparrot</code> 包含此堆栈中任何库的所有文件的数据集。由于数据集的太大，我们希望避免下载它；因此反，我们将使用流功能来动态过滤它。为了使用前面提到的库过滤代码示例，我们将使用以下函数：",nl,Gl,xl,H,is="让我们用两个例子来测试一下：",Ml,gl,Fl,S,Yl,q,Te="我们可以使用它来创建一个函数来流式传输数据集并过滤我们想要的元素：",Hl,D,Sl,L,we="然后我们可以简单地将此函数应用于流数据集：",ql,K,Dl,P,Ll,O,ue="这给我们留下了大约 3% 的原始数据集，这个数据集仍然相当可观——结果数据集有 6 GB，包含 600,000 个 Python 脚本！过滤完整数据集可能需要 2-3 小时，具体取决于您的机器和带宽。如果您不想自己经历这个漫长的过程，我们在 Hub 上提供过滤后的数据集供您下载：",Kl,ll,Pl,sl,Js,al,pl,yl,Bl="让我们看一个来自数据集的例子。我们将只显示每个字段的前 200 个字符：",Ol,el,os,rl,cl,il,Xl="我们可以看到 <code>content</code> 字段包含我们希望我们的模型训练的代码。现在我们有了一个数据集，我们需要预处理文本，使其采用适合预训练的格式。",Us,Jl,ol,Ul,ml,Tl,Rl="第一步是对数据进行标记，以便我们可以将其用于训练。由于我们的目标主要是自动完成短函数调用，因此我们可以保持上下文大小相对较小。这样做的好处是我们可以更快地训练模型并且它需要的内存显着减少。如果您的应用程序拥有更多上下文很重要（例如，如果您希望模型基于具有函数定义的文件编写单元测试），请确保增加该数量，但请记住，这需要更大的 GPU 内存占用。现在，让我们将上下文大小固定为 128 个标记，而不是 GPT-2 或 GPT-3 中分别使用的 1,024 或 2,048 个标记。",ms,wl,Ql='大多数文档包含超过 128 个标记，因此简单地将输入截断到最大长度将消除我们数据集的很大一部分。相反，我们将使用 <code>return_overflowing_tokens</code> 标记整个输入并将其分成几个块的选项，就像我们在<a href="/course/chapter6/4">第六章</a>. 我们还将使用 <code>return_length</code> 选项自动返回每个创建的块的长度。通常最后一个块会小于上下文大小，我们会去掉这些块以避免填充问题；因为无论如何我们都有大量数据。',ls,E,de='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/chunking_texts.svg" alt="Chunking a large texts in several pieces."/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/chunking_texts-dark.svg" alt="Chunking a large texts in several pieces."/>',ss,tl,Ts="让我们通过查看前两个示例来确切了解这是如何工作的：",ul,dl,s,U,be,ws,mt="我们可以看 到，从这两个示例中我们总共得到了 34 个片段。查看块长度，我们可以看到两个文档末尾的块都少于 128 个标记（分别为 117 和 41）。这些仅代表我们拥有的数据集的一小部分，因此我们可以安全地将它们扔掉。通过 <code>overflow_to_sample_mapping</code> 字段，我们还可以重建哪些块属于哪些输入样本。",fe,us,Tt='通过这个操作，我们使用了一个方便的🤗 Datasets 中的<code>Dataset.map()</code> 函数，就是不需要一对一的映射；正如我们在<a href="/course/chapter7/3">第三节</a>,我们可以创建具有比输入批次更多或更少元素的批次。这在执行更改元素数量的数据增强或数据过滤等操作时非常有用。在我们的例子中，当将每个元素标记为指定上下文大小的块时，我们从每个文档中创建了许多样本。我们只需要确保删除现有的列，因为它们的大小存在冲突。如果我们想保留它们，我们可以适当地重复它们，并在<code>Dataset.map()</code> 调用中返回它们:',Ce,ds,he,js,Ie,bs,wt="我们现在有 1670 万个示例，每个示例有 128 个tokens ，总共相当于大约 21 亿个tokens 。作为参考，OpenAI 的 GPT-3 和 Codex 模型分别在 300 和 1000 亿个tokens 上训练，其中 Codex 模型从 GPT-3 检查点初始化。我们在本节中的目标不是与这些模型竞争，这些模型可以生成长而连贯的文本，而是创建一个缩小版本，为数据科学家提供快速自动完成功能。",_e,fs,ut="现在我们已经准备好了数据集，让我们设置模型！",ke,ts,Ze,Cs,$e,hs,dt="我们的第一步是新初始化一个 GPT-2 模型。我们将对我们的模型使用与小型 GPT-2 模型相同的配置，因此我们加载预训练配置，确保分词器大小与模型词汇量大小匹配并设置 <code>bos</code> 和 <code>eos</code> （序列的开始和结束）令牌 ID：",Ge,jl,bl,te,Is,jt="我们的模型有 1.24 亿个参数，我们必须对其进行调整。在开始训练之前，我们需要设置一个负责创建批次的数据整理器。我们可以使用 <code>DataCollatorForLanguageModeling</code> ，它是专为语言建模而设计（顾名思义）。除了堆叠和填充批次，它还负责创建语言模型标签——在因果语言建模中，输入也用作标签（只是移动了一个元素），并且这个数据整理器在训练期间即时创建它们，所以我们不需要复制 <code>input_ids</code>。",ge,_s,bt="注意 <code>DataCollatorForLanguageModeling</code> 支持掩码语言建模 (MLM) 和因果语言建模 (CLM)。默认情况下它为 MLM 准备数据，但我们可以通过设置<code>mlm=False</code>参数切换到 CLM ：",Be,fl,Cl,ae,ks,ft="让我们看一个例子：",Xe,Zs,Re,hl,Il,ne,$s,Ct="我们可以看到示例已经堆叠在一起，并且所有张量都具有相同的形状。",Qe,Me,as,We,Gs,ht="现在我们已经准备好实际训练我们的模型的一切了——毕竟这不是那么多工作！在我们开始训练之前，我们应该登录 Hugging Face。如果您在笔记本上工作，则可以使用以下实用程序功能：",Ve,gs,Ee,Bs,It="这将显示一个小部件，您可以在其中输入您的 Hugging Face 登录凭据。",ze,Xs,_t="如果您不是在notebook上工作，只需在终端中输入以下行：",Ae,Rs,ve,_l,kl,pe,ns,Ne,Ms,xe,Qs,Fe,Ws,kt="现在是关键的部分：让我们看看经过训练的模型的实际效果如何！我们可以在日志中看到损失稳步下降，但为了让模型进行测试，让我们看看它在某些测试上的表现如何。为此，我们将模型包装在文本生成中的<code>pipeline</code> ，如果有可用的，我们会将它放在 GPU 上进行快速生成：",Ye,Zl,$l,ye,Vs,Zt="让我们从创建散点图的简单任务开始：",He,Es,Se,zs,qe,As,$t="结果看起来是正确的。它也适用于 <code>pandas</code> 类型？让我们看看我们是否使用两个数组可以创建一个 <code>DataFrame</code> ：",De,vs,Le,Ns,Ke,xs,Gt="很好，这是正确的答案——尽管它随后再次插入了列 <code>x</code> 。由于生成的token数量有限，以下 <code>for</code> 循环被切断。让我们看看我们是否可以做一些更复杂的事情并让模型帮助我们分组操作：",Pe,Fs,Oe,Ys,lt,Hs,gt="不错;这是正确的做法。最后，让我们看看我们是否也可以将其用于 <code>scikit-learn</code> 并建立一个随机森林模型：",st,Ss,et,qs,tt,re,ce,Ds,at,je,nt;_=new ya({props:{fw:I[0]}}),G=new ie({props:{title:"从头开始训练因果语言模型",local:"从头开始训练因果语言模型",headingTag:"h1"}});const Bt=[ia,ca],Ls=[];function Xt(l,M){return l[0]==="pt"?0:1}Z=Xt(I),o=Ls[Z]=Bt[Z](I),A=new Ut({props:{id:"Vpjb1lu0MDk"}}),x=new ie({props:{title:"收集数据",local:"收集数据",headingTag:"h2"}}),Gl=new h({props:{code:"ZGVmJTIwYW55X2tleXdvcmRfaW5fc3RyaW5nKHN0cmluZyUyQyUyMGtleXdvcmRzKSUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMGtleXdvcmQlMjBpbiUyMGtleXdvcmRzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBrZXl3b3JkJTIwaW4lMjBzdHJpbmclM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBUcnVlJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwRmFsc2U=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">any_keyword_in_string</span>(<span class="hljs-params">string, keywords</span>):
    <span class="hljs-keyword">for</span> keyword <span class="hljs-keyword">in</span> keywords:
        <span class="hljs-keyword">if</span> keyword <span class="hljs-keyword">in</span> string:
            <span class="hljs-keyword">return</span> <span class="hljs-literal">True</span>
    <span class="hljs-keyword">return</span> <span class="hljs-literal">False</span>`,wrap:!1}}),gl=new h({props:{code:"ZmlsdGVycyUyMCUzRCUyMCU1QiUyMnBhbmRhcyUyMiUyQyUyMCUyMnNrbGVhcm4lMjIlMkMlMjAlMjJtYXRwbG90bGliJTIyJTJDJTIwJTIyc2VhYm9ybiUyMiU1RCUwQWV4YW1wbGVfMSUyMCUzRCUyMCUyMmltcG9ydCUyMG51bXB5JTIwYXMlMjBucCUyMiUwQWV4YW1wbGVfMiUyMCUzRCUyMCUyMmltcG9ydCUyMHBhbmRhcyUyMGFzJTIwcGQlMjIlMEElMEFwcmludCglMEElMjAlMjAlMjAlMjBhbnlfa2V5d29yZF9pbl9zdHJpbmcoZXhhbXBsZV8xJTJDJTIwZmlsdGVycyklMkMlMjBhbnlfa2V5d29yZF9pbl9zdHJpbmcoZXhhbXBsZV8yJTJDJTIwZmlsdGVycyklMEEp",highlighted:`filters = [<span class="hljs-string">&quot;pandas&quot;</span>, <span class="hljs-string">&quot;sklearn&quot;</span>, <span class="hljs-string">&quot;matplotlib&quot;</span>, <span class="hljs-string">&quot;seaborn&quot;</span>]
example_1 = <span class="hljs-string">&quot;import numpy as np&quot;</span>
example_2 = <span class="hljs-string">&quot;import pandas as pd&quot;</span>

<span class="hljs-built_in">print</span>(
    any_keyword_in_string(example_1, filters), any_keyword_in_string(example_2, filters)
)`,wrap:!1}}),S=new h({props:{code:"RmFsc2UlMjBUcnVl",highlighted:'<span class="hljs-literal">False</span> <span class="hljs-literal">True</span>',wrap:!1}}),D=new h({props:{code:"ZnJvbSUyMGNvbGxlY3Rpb25zJTIwaW1wb3J0JTIwZGVmYXVsdGRpY3QlMEFmcm9tJTIwdHFkbSUyMGltcG9ydCUyMHRxZG0lMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBEYXRhc2V0JTBBJTBBJTBBZGVmJTIwZmlsdGVyX3N0cmVhbWluZ19kYXRhc2V0KGRhdGFzZXQlMkMlMjBmaWx0ZXJzKSUzQSUwQSUyMCUyMCUyMCUyMGZpbHRlcmVkX2RpY3QlMjAlM0QlMjBkZWZhdWx0ZGljdChsaXN0KSUwQSUyMCUyMCUyMCUyMHRvdGFsJTIwJTNEJTIwMCUwQSUyMCUyMCUyMCUyMGZvciUyMHNhbXBsZSUyMGluJTIwdHFkbShpdGVyKGRhdGFzZXQpKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvdGFsJTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwYW55X2tleXdvcmRfaW5fc3RyaW5nKHNhbXBsZSU1QiUyMmNvbnRlbnQlMjIlNUQlMkMlMjBmaWx0ZXJzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGslMkMlMjB2JTIwaW4lMjBzYW1wbGUuaXRlbXMoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZpbHRlcmVkX2RpY3QlNUJrJTVELmFwcGVuZCh2KSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlN0JsZW4oZmlsdGVyZWRfZGljdCU1Qidjb250ZW50JyU1RCklMkZ0b3RhbCUzQS4yJTI1JTdEJTIwb2YlMjBkYXRhJTIwYWZ0ZXIlMjBmaWx0ZXJpbmcuJTIyKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMERhdGFzZXQuZnJvbV9kaWN0KGZpbHRlcmVkX2RpY3Qp",highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset


<span class="hljs-keyword">def</span> <span class="hljs-title function_">filter_streaming_dataset</span>(<span class="hljs-params">dataset, filters</span>):
    filtered_dict = defaultdict(<span class="hljs-built_in">list</span>)
    total = <span class="hljs-number">0</span>
    <span class="hljs-keyword">for</span> sample <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">iter</span>(dataset)):
        total += <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> any_keyword_in_string(sample[<span class="hljs-string">&quot;content&quot;</span>], filters):
            <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> sample.items():
                filtered_dict[k].append(v)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{<span class="hljs-built_in">len</span>(filtered_dict[<span class="hljs-string">&#x27;content&#x27;</span>])/total:<span class="hljs-number">.2</span>%}</span> of data after filtering.&quot;</span>)
    <span class="hljs-keyword">return</span> Dataset.from_dict(filtered_dict)`,wrap:!1}}),K=new h({props:{code:"JTIzJTIwVGhpcyUyMGNlbGwlMjB3aWxsJTIwdGFrZSUyMGElMjB2ZXJ5JTIwbG9uZyUyMHRpbWUlMjB0byUyMGV4ZWN1dGUlMkMlMjBzbyUyMHlvdSUyMHNob3VsZCUyMHNraXAlMjBpdCUyMGFuZCUyMGdvJTIwdG8lMEElMjMlMjB0aGUlMjBuZXh0JTIwb25lISUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQXNwbGl0JTIwJTNEJTIwJTIydHJhaW4lMjIlMjAlMjAlMjMlMjAlMjJ2YWxpZCUyMiUwQWZpbHRlcnMlMjAlM0QlMjAlNUIlMjJwYW5kYXMlMjIlMkMlMjAlMjJza2xlYXJuJTIyJTJDJTIwJTIybWF0cGxvdGxpYiUyMiUyQyUyMCUyMnNlYWJvcm4lMjIlNUQlMEElMEFkYXRhJTIwJTNEJTIwbG9hZF9kYXRhc2V0KGYlMjJ0cmFuc2Zvcm1lcnNib29rJTJGY29kZXBhcnJvdC0lN0JzcGxpdCU3RCUyMiUyQyUyMHNwbGl0JTNEc3BsaXQlMkMlMjBzdHJlYW1pbmclM0RUcnVlKSUwQWZpbHRlcmVkX2RhdGElMjAlM0QlMjBmaWx0ZXJfc3RyZWFtaW5nX2RhdGFzZXQoZGF0YSUyQyUyMGZpbHRlcnMp",highlighted:`<span class="hljs-comment"># This cell will take a very long time to execute, so you should skip it and go to</span>
<span class="hljs-comment"># the next one!</span>
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

split = <span class="hljs-string">&quot;train&quot;</span>  <span class="hljs-comment"># &quot;valid&quot;</span>
filters = [<span class="hljs-string">&quot;pandas&quot;</span>, <span class="hljs-string">&quot;sklearn&quot;</span>, <span class="hljs-string">&quot;matplotlib&quot;</span>, <span class="hljs-string">&quot;seaborn&quot;</span>]

data = load_dataset(<span class="hljs-string">f&quot;transformersbook/codeparrot-<span class="hljs-subst">{split}</span>&quot;</span>, split=split, streaming=<span class="hljs-literal">True</span>)
filtered_data = filter_streaming_dataset(data, filters)`,wrap:!1}}),P=new h({props:{code:"My4yNiUyNSUyMG9mJTIwZGF0YSUyMGFmdGVyJTIwZmlsdGVyaW5nLg==",highlighted:'<span class="hljs-number">3.26</span>% of data after filtering.',wrap:!1}}),ll=new h({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTJDJTIwRGF0YXNldERpY3QlMEElMEFkc190cmFpbiUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJodWdnaW5nZmFjZS1jb3Vyc2UlMkZjb2RlcGFycm90LWRzLXRyYWluJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEFkc192YWxpZCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJodWdnaW5nZmFjZS1jb3Vyc2UlMkZjb2RlcGFycm90LWRzLXZhbGlkJTIyJTJDJTIwc3BsaXQlM0QlMjJ2YWxpZGF0aW9uJTIyKSUwQSUwQXJhd19kYXRhc2V0cyUyMCUzRCUyMERhdGFzZXREaWN0KCUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRyYWluJTIyJTNBJTIwZHNfdHJhaW4lMkMlMjAlMjAlMjMlMjAuc2h1ZmZsZSgpLnNlbGVjdChyYW5nZSg1MDAwMCkpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydmFsaWQlMjIlM0ElMjBkc192YWxpZCUyQyUyMCUyMCUyMyUyMC5zaHVmZmxlKCkuc2VsZWN0KHJhbmdlKDUwMCkpJTBBJTIwJTIwJTIwJTIwJTdEJTBBKSUwQSUwQXJhd19kYXRhc2V0cw==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, DatasetDict

ds_train = load_dataset(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds-train&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
ds_valid = load_dataset(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds-valid&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)

raw_datasets = DatasetDict(
    {
        <span class="hljs-string">&quot;train&quot;</span>: ds_train,  <span class="hljs-comment"># .shuffle().select(range(50000)),</span>
        <span class="hljs-string">&quot;valid&quot;</span>: ds_valid,  <span class="hljs-comment"># .shuffle().select(range(500))</span>
    }
)

raw_datasets`,wrap:!1}}),sl=new h({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ3JlcG9fbmFtZSclMkMlMjAncGF0aCclMkMlMjAnY29waWVzJyUyQyUyMCdzaXplJyUyQyUyMCdjb250ZW50JyUyQyUyMCdsaWNlbnNlJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNjA2NzIwJTBBJTIwJTIwJTIwJTIwJTdEKSUwQSUyMCUyMCUyMCUyMHZhbGlkJTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidyZXBvX25hbWUnJTJDJTIwJ3BhdGgnJTJDJTIwJ2NvcGllcyclMkMlMjAnc2l6ZSclMkMlMjAnY29udGVudCclMkMlMjAnbGljZW5zZSclNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBudW1fcm93cyUzQSUyMDMzMjIlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTdEKQ==",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;repo_name&#x27;</span>, <span class="hljs-string">&#x27;path&#x27;</span>, <span class="hljs-string">&#x27;copies&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>, <span class="hljs-string">&#x27;license&#x27;</span>],
        num_rows: <span class="hljs-number">606720</span>
    })
    valid: Dataset({
        features: [<span class="hljs-string">&#x27;repo_name&#x27;</span>, <span class="hljs-string">&#x27;path&#x27;</span>, <span class="hljs-string">&#x27;copies&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>, <span class="hljs-string">&#x27;license&#x27;</span>],
        num_rows: <span class="hljs-number">3322</span>
    })
})`,wrap:!1}}),al=new rs({props:{$$slots:{default:[Ja]},$$scope:{ctx:I}}}),el=new h({props:{code:"Zm9yJTIwa2V5JTIwaW4lMjByYXdfZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQlM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTdCa2V5LnVwcGVyKCklN0QlM0ElMjAlN0JyYXdfZGF0YXNldHMlNUIndHJhaW4nJTVEJTVCMCU1RCU1QmtleSU1RCU1QiUzQTIwMCU1RCU3RCUyMik=",highlighted:`<span class="hljs-keyword">for</span> key <span class="hljs-keyword">in</span> raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key.upper()}</span>: <span class="hljs-subst">{raw_datasets[<span class="hljs-string">&#x27;train&#x27;</span>][<span class="hljs-number">0</span>][key][:<span class="hljs-number">200</span>]}</span>&quot;</span>)`,wrap:!1}}),rl=new h({props:{code:"J1JFUE9fTkFNRSUzQSUyMGttaWtlJTJGc2Npa2l0LWxlYXJuJyUwQSdQQVRIJTNBJTIwc2tsZWFybiUyRnV0aWxzJTJGX19pbml0X18ucHknJTBBJ0NPUElFUyUzQSUyMDMnJTBBJ1NJWkUlM0ElMjAxMDA5NCclMEEnJydDT05URU5UJTNBJTIwJTIyJTIyJTIyJTBBVGhlJTIwJTNBbW9kJTNBJTYwc2tsZWFybi51dGlscyU2MCUyMG1vZHVsZSUyMGluY2x1ZGVzJTIwdmFyaW91cyUyMHV0aWxpdGVzLiUwQSUyMiUyMiUyMiUwQSUwQWZyb20lMjBjb2xsZWN0aW9ucyUyMGltcG9ydCUyMFNlcXVlbmNlJTBBJTBBaW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBZnJvbSUyMHNjaXB5LnNwYXJzZSUyMGltcG9ydCUyMGlzc3BhcnNlJTBBaW1wb3J0JTIwd2FybmluZ3MlMEElMEFmcm9tJTIwLm11cm11cmhhc2glMjBpbXBvcnQlMjBtdXJtJTBBTElDRU5TRSUzQSUyMGJzZC0zLWNsYXVzZScnJw==",highlighted:`<span class="hljs-string">&#x27;REPO_NAME: kmike/scikit-learn&#x27;</span>
<span class="hljs-string">&#x27;PATH: sklearn/utils/__init__.py&#x27;</span>
<span class="hljs-string">&#x27;COPIES: 3&#x27;</span>
<span class="hljs-string">&#x27;SIZE: 10094&#x27;</span>
<span class="hljs-string">&#x27;&#x27;&#x27;CONTENT: &quot;&quot;&quot;
The :mod:\`sklearn.utils\` module includes various utilites.
&quot;&quot;&quot;

from collections import Sequence

import numpy as np
from scipy.sparse import issparse
import warnings

from .murmurhash import murm
LICENSE: bsd-3-clause&#x27;&#x27;&#x27;</span>`,wrap:!1}}),Jl=new ie({props:{title:"准备数据集",local:"准备数据集",headingTag:"h2"}}),Ul=new Ut({props:{id:"ma1TrR7gE7I"}}),dl=new h({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjb250ZXh0X2xlbmd0aCUyMCUzRCUyMDEyOCUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmNvZGUtc2VhcmNoLW5ldC10b2tlbml6ZXIlMjIpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMHRva2VuaXplciglMEElMjAlMjAlMjAlMjByYXdfZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RCU1QiUzQTIlNUQlNUIlMjJjb250ZW50JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEY29udGV4dF9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fb3ZlcmZsb3dpbmdfdG9rZW5zJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHJldHVybl9sZW5ndGglM0RUcnVlJTJDJTBBKSUwQSUwQXByaW50KGYlMjJJbnB1dCUyMElEcyUyMGxlbmd0aCUzQSUyMCU3QmxlbihvdXRwdXRzJTVCJ2lucHV0X2lkcyclNUQpJTdEJTIyKSUwQXByaW50KGYlMjJJbnB1dCUyMGNodW5rJTIwbGVuZ3RocyUzQSUyMCU3QihvdXRwdXRzJTVCJ2xlbmd0aCclNUQpJTdEJTIyKSUwQXByaW50KGYlMjJDaHVuayUyMG1hcHBpbmclM0ElMjAlN0JvdXRwdXRzJTVCJ292ZXJmbG93X3RvX3NhbXBsZV9tYXBwaW5nJyU1RCU3RCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

context_length = <span class="hljs-number">128</span>
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggingface-course/code-search-net-tokenizer&quot;</span>)

outputs = tokenizer(
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">2</span>][<span class="hljs-string">&quot;content&quot;</span>],
    truncation=<span class="hljs-literal">True</span>,
    max_length=context_length,
    return_overflowing_tokens=<span class="hljs-literal">True</span>,
    return_length=<span class="hljs-literal">True</span>,
)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Input IDs length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(outputs[<span class="hljs-string">&#x27;input_ids&#x27;</span>])}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Input chunk lengths: <span class="hljs-subst">{(outputs[<span class="hljs-string">&#x27;length&#x27;</span>])}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Chunk mapping: <span class="hljs-subst">{outputs[<span class="hljs-string">&#x27;overflow_to_sample_mapping&#x27;</span>]}</span>&quot;</span>)`,wrap:!1}}),U=new h({props:{code:"SW5wdXQlMjBJRHMlMjBsZW5ndGglM0ElMjAzNCUwQUlucHV0JTIwY2h1bmslMjBsZW5ndGhzJTNBJTIwJTVCMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTE3JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwNDElNUQlMEFDaHVuayUyMG1hcHBpbmclM0ElMjAlNUIwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTVE",highlighted:`Input IDs length: <span class="hljs-number">34</span>
Input chunk lengths: [<span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">117</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">41</span>]
Chunk mapping: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]`,wrap:!1}}),ds=new h({props:{code:"ZGVmJTIwdG9rZW5pemUoZWxlbWVudCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsZW1lbnQlNUIlMjJjb250ZW50JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEY29udGV4dF9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm5fb3ZlcmZsb3dpbmdfdG9rZW5zJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybl9sZW5ndGglM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMGlucHV0X2JhdGNoJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwbGVuZ3RoJTJDJTIwaW5wdXRfaWRzJTIwaW4lMjB6aXAob3V0cHV0cyU1QiUyMmxlbmd0aCUyMiU1RCUyQyUyMG91dHB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBsZW5ndGglMjAlM0QlM0QlMjBjb250ZXh0X2xlbmd0aCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X2JhdGNoLmFwcGVuZChpbnB1dF9pZHMpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTIyaW5wdXRfaWRzJTIyJTNBJTIwaW5wdXRfYmF0Y2glN0QlMEElMEElMEF0b2tlbml6ZWRfZGF0YXNldHMlMjAlM0QlMjByYXdfZGF0YXNldHMubWFwKCUwQSUyMCUyMCUyMCUyMHRva2VuaXplJTJDJTIwYmF0Y2hlZCUzRFRydWUlMkMlMjByZW1vdmVfY29sdW1ucyUzRHJhd19kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVELmNvbHVtbl9uYW1lcyUwQSklMEF0b2tlbml6ZWRfZGF0YXNldHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">element</span>):
    outputs = tokenizer(
        element[<span class="hljs-string">&quot;content&quot;</span>],
        truncation=<span class="hljs-literal">True</span>,
        max_length=context_length,
        return_overflowing_tokens=<span class="hljs-literal">True</span>,
        return_length=<span class="hljs-literal">True</span>,
    )
    input_batch = []
    <span class="hljs-keyword">for</span> length, input_ids <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(outputs[<span class="hljs-string">&quot;length&quot;</span>], outputs[<span class="hljs-string">&quot;input_ids&quot;</span>]):
        <span class="hljs-keyword">if</span> length == context_length:
            input_batch.append(input_ids)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;input_ids&quot;</span>: input_batch}


tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>(
    tokenize, batched=<span class="hljs-literal">True</span>, remove_columns=raw_datasets[<span class="hljs-string">&quot;train&quot;</span>].column_names
)
tokenized_datasets`,wrap:!1}}),js=new h({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2lucHV0X2lkcyclNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBudW1fcm93cyUzQSUyMDE2NzAyMDYxJTBBJTIwJTIwJTIwJTIwJTdEKSUwQSUyMCUyMCUyMCUyMHZhbGlkJTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidpbnB1dF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjA5MzE2NCUwQSUyMCUyMCUyMCUyMCU3RCklMEElN0Qp",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;input_ids&#x27;</span>],
        num_rows: <span class="hljs-number">16702061</span>
    })
    valid: Dataset({
        features: [<span class="hljs-string">&#x27;input_ids&#x27;</span>],
        num_rows: <span class="hljs-number">93164</span>
    })
})`,wrap:!1}}),ts=new rs({props:{$$slots:{default:[oa]},$$scope:{ctx:I}}}),Cs=new ie({props:{title:"初始化新模型",local:"初始化新模型",headingTag:"h2"}});const Rt=[ma,Ua],Ks=[];function Qt(l,M){return l[0]==="pt"?0:1}jl=Qt(I),bl=Ks[jl]=Rt[jl](I);const Wt=[wa,Ta],Ps=[];function Vt(l,M){return l[0]==="pt"?0:1}fl=Vt(I),Cl=Ps[fl]=Wt[fl](I),Zs=new h({props:{code:"b3V0JTIwJTNEJTIwZGF0YV9jb2xsYXRvciglNUJ0b2tlbml6ZWRfZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTVCaSU1RCUyMGZvciUyMGklMjBpbiUyMHJhbmdlKDUpJTVEKSUwQWZvciUyMGtleSUyMGluJTIwb3V0JTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiU3QmtleSU3RCUyMHNoYXBlJTNBJTIwJTdCb3V0JTVCa2V5JTVELnNoYXBlJTdEJTIyKQ==",highlighted:`out = data_collator([tokenized_dataset[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>)])
<span class="hljs-keyword">for</span> key <span class="hljs-keyword">in</span> out:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span> shape: <span class="hljs-subst">{out[key].shape}</span>&quot;</span>)`,wrap:!1}});const Et=[da,ua],Os=[];function zt(l,M){return l[0]==="pt"?0:1}hl=zt(I),Il=Os[hl]=Et[hl](I);let Q=I[0]==="tf"&&Pt();as=new rs({props:{warning:!0,$$slots:{default:[ja]},$$scope:{ctx:I}}}),gs=new h({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),Rs=new h({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:"huggingface-cli login",wrap:!1}});const At=[fa,ba],le=[];function vt(l,M){return l[0]==="pt"?0:1}_l=vt(I),kl=le[_l]=At[_l](I),ns=new rs({props:{$$slots:{default:[Ca]},$$scope:{ctx:I}}}),Ms=new rs({props:{$$slots:{default:[_a]},$$scope:{ctx:I}}}),Qs=new ie({props:{title:"使用管道生成代码",local:"使用管道生成代码",headingTag:"h2"}});const Nt=[Za,ka],se=[];function xt(l,M){return l[0]==="pt"?0:1}Zl=xt(I),$l=se[Zl]=Nt[Zl](I),Es=new h({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTVDJTBBJTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBzY2F0dGVyJTIwcGxvdCUyMHdpdGglMjB4JTJDJTIweSUwQSUyMiUyMiUyMiUwQXByaW50KHBpcGUodHh0JTJDJTIwbnVtX3JldHVybl9zZXF1ZW5jZXMlM0QxKSU1QjAlNUQlNUIlMjJnZW5lcmF0ZWRfdGV4dCUyMiU1RCk=",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;\\
# create some data
x = np.random.randn(100)
y = np.random.randn(100)

# create scatter plot with x, y
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),zs=new h({props:{code:"JTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBzY2F0dGVyJTIwcGxvdCUyMHdpdGglMjB4JTJDJTIweSUwQXBsdC5zY2F0dGVyKHglMkMlMjB5KSUwQSUwQSUyMyUyMGNyZWF0ZSUyMHNjYXR0ZXI=",highlighted:`<span class="hljs-comment"># create some data</span>
x = np.random.randn(<span class="hljs-number">100</span>)
y = np.random.randn(<span class="hljs-number">100</span>)

<span class="hljs-comment"># create scatter plot with x, y</span>
plt.scatter(x, y)

<span class="hljs-comment"># create scatter</span>`,wrap:!1}}),vs=new h({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTVDJTBBJTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBkYXRhZnJhbWUlMjBmcm9tJTIweCUyMGFuZCUyMHklMEElMjIlMjIlMjIlMEFwcmludChwaXBlKHR4dCUyQyUyMG51bV9yZXR1cm5fc2VxdWVuY2VzJTNEMSklNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQp",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;\\
# create some data
x = np.random.randn(100)
y = np.random.randn(100)

# create dataframe from x and y
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),Ns=new h({props:{code:"JTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBkYXRhZnJhbWUlMjBmcm9tJTIweCUyMGFuZCUyMHklMEFkZiUyMCUzRCUyMHBkLkRhdGFGcmFtZSglN0IneCclM0ElMjB4JTJDJTIwJ3knJTNBJTIweSU3RCklMEFkZi5pbnNlcnQoMCUyQyd4JyUyQyUyMHgpJTBBZm9y",highlighted:`<span class="hljs-comment"># create some data</span>
x = np.random.randn(<span class="hljs-number">100</span>)
y = np.random.randn(<span class="hljs-number">100</span>)

<span class="hljs-comment"># create dataframe from x and y</span>
df = pd.DataFrame({<span class="hljs-string">&#x27;x&#x27;</span>: x, <span class="hljs-string">&#x27;y&#x27;</span>: y})
df.insert(<span class="hljs-number">0</span>,<span class="hljs-string">&#x27;x&#x27;</span>, x)
<span class="hljs-keyword">for</span>`,wrap:!1}}),Fs=new h({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTVDJTBBJTIzJTIwZGF0YWZyYW1lJTIwd2l0aCUyMHByb2Zlc3Npb24lMkMlMjBpbmNvbWUlMjBhbmQlMjBuYW1lJTBBZGYlMjAlM0QlMjBwZC5EYXRhRnJhbWUoJTdCJ3Byb2Zlc3Npb24nJTNBJTIweCUyQyUyMCdpbmNvbWUnJTNBeSUyQyUyMCduYW1lJyUzQSUyMHolN0QpJTBBJTBBJTIzJTIwY2FsY3VsYXRlJTIwdGhlJTIwbWVhbiUyMGluY29tZSUyMHBlciUyMHByb2Zlc3Npb24lMEElMjIlMjIlMjIlMEFwcmludChwaXBlKHR4dCUyQyUyMG51bV9yZXR1cm5fc2VxdWVuY2VzJTNEMSklNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQp",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;\\
# dataframe with profession, income and name
df = pd.DataFrame({&#x27;profession&#x27;: x, &#x27;income&#x27;:y, &#x27;name&#x27;: z})

# calculate the mean income per profession
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),Ys=new h({props:{code:"JTIzJTIwZGF0YWZyYW1lJTIwd2l0aCUyMHByb2Zlc3Npb24lMkMlMjBpbmNvbWUlMjBhbmQlMjBuYW1lJTBBZGYlMjAlM0QlMjBwZC5EYXRhRnJhbWUoJTdCJ3Byb2Zlc3Npb24nJTNBJTIweCUyQyUyMCdpbmNvbWUnJTNBeSUyQyUyMCduYW1lJyUzQSUyMHolN0QpJTBBJTBBJTIzJTIwY2FsY3VsYXRlJTIwdGhlJTIwbWVhbiUyMGluY29tZSUyMHBlciUyMHByb2Zlc3Npb24lMEFwcm9mZXNzaW9uJTIwJTNEJTIwZGYuZ3JvdXBieSglNUIncHJvZmVzc2lvbiclNUQpLm1lYW4oKSUwQSUwQSUyMyUyMGNvbXB1dGUlMjB0aGU=",highlighted:`<span class="hljs-comment"># dataframe with profession, income and name</span>
df = pd.DataFrame({<span class="hljs-string">&#x27;profession&#x27;</span>: x, <span class="hljs-string">&#x27;income&#x27;</span>:y, <span class="hljs-string">&#x27;name&#x27;</span>: z})

<span class="hljs-comment"># calculate the mean income per profession</span>
profession = df.groupby([<span class="hljs-string">&#x27;profession&#x27;</span>]).mean()

<span class="hljs-comment"># compute the</span>`,wrap:!1}}),Ss=new h({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTBBJTIzJTIwaW1wb3J0JTIwcmFuZG9tJTIwZm9yZXN0JTIwcmVncmVzc29yJTIwZnJvbSUyMHNjaWtpdC1sZWFybiUwQWZyb20lMjBza2xlYXJuLmVuc2VtYmxlJTIwaW1wb3J0JTIwUmFuZG9tRm9yZXN0UmVncmVzc29yJTBBJTBBJTIzJTIwZml0JTIwcmFuZG9tJTIwZm9yZXN0JTIwbW9kZWwlMjB3aXRoJTIwMzAwJTIwZXN0aW1hdG9ycyUyMG9uJTIwWCUyQyUyMHklM0ElMEElMjIlMjIlMjIlMEFwcmludChwaXBlKHR4dCUyQyUyMG51bV9yZXR1cm5fc2VxdWVuY2VzJTNEMSklNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQp",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;
# import random forest regressor from scikit-learn
from sklearn.ensemble import RandomForestRegressor

# fit random forest model with 300 estimators on X, y:
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),qs=new h({props:{code:"JTIzJTIwaW1wb3J0JTIwcmFuZG9tJTIwZm9yZXN0JTIwcmVncmVzc29yJTIwZnJvbSUyMHNjaWtpdC1sZWFybiUwQWZyb20lMjBza2xlYXJuLmVuc2VtYmxlJTIwaW1wb3J0JTIwUmFuZG9tRm9yZXN0UmVncmVzc29yJTBBJTBBJTIzJTIwZml0JTIwcmFuZG9tJTIwZm9yZXN0JTIwbW9kZWwlMjB3aXRoJTIwMzAwJTIwZXN0aW1hdG9ycyUyMG9uJTIwWCUyQyUyMHklM0ElMEFyZiUyMCUzRCUyMFJhbmRvbUZvcmVzdFJlZ3Jlc3NvcihuX2VzdGltYXRvcnMlM0QzMDAlMkMlMjByYW5kb21fc3RhdGUlM0RyYW5kb21fc3RhdGUlMkMlMjBtYXhfZGVwdGglM0QzKSUwQXJmLmZpdChYJTJDJTIweSklMEFyZg==",highlighted:`<span class="hljs-comment"># import random forest regressor from scikit-learn</span>
<span class="hljs-keyword">from</span> sklearn.ensemble <span class="hljs-keyword">import</span> RandomForestRegressor

<span class="hljs-comment"># fit random forest model with 300 estimators on X, y:</span>
rf = RandomForestRegressor(n_estimators=<span class="hljs-number">300</span>, random_state=random_state, max_depth=<span class="hljs-number">3</span>)
rf.fit(X, y)
rf`,wrap:!1}});function Ft(l,M){return l[0]==="tf"?Ga:$a}let Mt=Ft(I),es=Mt(I),W=I[0]==="pt"&&Ot(I);return Ds=new ra({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-CN/chapter7/6.mdx"}}),{c(){e=d("meta"),r=p(),n=d("p"),J=p(),m(_.$$.fragment),f=p(),m(G.$$.fragment),$=p(),o.c(),C=p(),g=d("p"),g.innerHTML=z,V=p(),R=d("p"),R.innerHTML=k,B=p(),m(A.$$.fragment),Vl=p(),v=d("p"),v.innerHTML=Je,El=p(),X=d("iframe"),zl=p(),N=d("p"),N.innerHTML=Ue,Al=p(),m(x.$$.fragment),vl=p(),F=d("p"),F.innerHTML=me,Nl=p(),Y=d("p"),Y.innerHTML=cs,nl=p(),m(Gl.$$.fragment),xl=p(),H=d("p"),H.textContent=is,Ml=p(),m(gl.$$.fragment),Fl=p(),m(S.$$.fragment),Yl=p(),q=d("p"),q.textContent=Te,Hl=p(),m(D.$$.fragment),Sl=p(),L=d("p"),L.textContent=we,ql=p(),m(K.$$.fragment),Dl=p(),m(P.$$.fragment),Ll=p(),O=d("p"),O.textContent=ue,Kl=p(),m(ll.$$.fragment),Pl=p(),m(sl.$$.fragment),Js=p(),m(al.$$.fragment),pl=p(),yl=d("p"),yl.textContent=Bl,Ol=p(),m(el.$$.fragment),os=p(),m(rl.$$.fragment),cl=p(),il=d("p"),il.innerHTML=Xl,Us=p(),m(Jl.$$.fragment),ol=p(),m(Ul.$$.fragment),ml=p(),Tl=d("p"),Tl.textContent=Rl,ms=p(),wl=d("p"),wl.innerHTML=Ql,ls=p(),E=d("div"),E.innerHTML=de,ss=p(),tl=d("p"),tl.textContent=Ts,ul=p(),m(dl.$$.fragment),s=p(),m(U.$$.fragment),be=p(),ws=d("p"),ws.innerHTML=mt,fe=p(),us=d("p"),us.innerHTML=Tt,Ce=p(),m(ds.$$.fragment),he=p(),m(js.$$.fragment),Ie=p(),bs=d("p"),bs.textContent=wt,_e=p(),fs=d("p"),fs.textContent=ut,ke=p(),m(ts.$$.fragment),Ze=p(),m(Cs.$$.fragment),$e=p(),hs=d("p"),hs.innerHTML=dt,Ge=p(),bl.c(),te=p(),Is=d("p"),Is.innerHTML=jt,ge=p(),_s=d("p"),_s.innerHTML=bt,Be=p(),Cl.c(),ae=p(),ks=d("p"),ks.textContent=ft,Xe=p(),m(Zs.$$.fragment),Re=p(),Il.c(),ne=p(),$s=d("p"),$s.textContent=Ct,Qe=p(),Q&&Q.c(),Me=p(),m(as.$$.fragment),We=p(),Gs=d("p"),Gs.textContent=ht,Ve=p(),m(gs.$$.fragment),Ee=p(),Bs=d("p"),Bs.textContent=It,ze=p(),Xs=d("p"),Xs.textContent=_t,Ae=p(),m(Rs.$$.fragment),ve=p(),kl.c(),pe=p(),m(ns.$$.fragment),Ne=p(),m(Ms.$$.fragment),xe=p(),m(Qs.$$.fragment),Fe=p(),Ws=d("p"),Ws.innerHTML=kt,Ye=p(),$l.c(),ye=p(),Vs=d("p"),Vs.textContent=Zt,He=p(),m(Es.$$.fragment),Se=p(),m(zs.$$.fragment),qe=p(),As=d("p"),As.innerHTML=$t,De=p(),m(vs.$$.fragment),Le=p(),m(Ns.$$.fragment),Ke=p(),xs=d("p"),xs.innerHTML=Gt,Pe=p(),m(Fs.$$.fragment),Oe=p(),m(Ys.$$.fragment),lt=p(),Hs=d("p"),Hs.innerHTML=gt,st=p(),m(Ss.$$.fragment),et=p(),m(qs.$$.fragment),tt=p(),es.c(),re=p(),W&&W.c(),ce=p(),m(Ds.$$.fragment),at=p(),je=d("p"),this.h()},l(l){const M=Ma("svelte-u9bgzb",document.head);e=j(M,"META",{name:!0,content:!0}),M.forEach(t),r=y(l),n=j(l,"P",{}),ot(n).forEach(t),J=y(l),T(_.$$.fragment,l),f=y(l),T(G.$$.fragment,l),$=y(l),o.l(l),C=y(l),g=j(l,"P",{"data-svelte-h":!0}),b(g)!=="svelte-4o1apo"&&(g.innerHTML=z),V=y(l),R=j(l,"P",{"data-svelte-h":!0}),b(R)!=="svelte-1hq6k7z"&&(R.innerHTML=k),B=y(l),T(A.$$.fragment,l),Vl=y(l),v=j(l,"P",{"data-svelte-h":!0}),b(v)!=="svelte-qzrkwd"&&(v.innerHTML=Je),El=y(l),X=j(l,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),ot(X).forEach(t),zl=y(l),N=j(l,"P",{"data-svelte-h":!0}),b(N)!=="svelte-xa1v3k"&&(N.innerHTML=Ue),Al=y(l),T(x.$$.fragment,l),vl=y(l),F=j(l,"P",{"data-svelte-h":!0}),b(F)!=="svelte-1jkjbul"&&(F.innerHTML=me),Nl=y(l),Y=j(l,"P",{"data-svelte-h":!0}),b(Y)!=="svelte-10krasy"&&(Y.innerHTML=cs),nl=y(l),T(Gl.$$.fragment,l),xl=y(l),H=j(l,"P",{"data-svelte-h":!0}),b(H)!=="svelte-188avx"&&(H.textContent=is),Ml=y(l),T(gl.$$.fragment,l),Fl=y(l),T(S.$$.fragment,l),Yl=y(l),q=j(l,"P",{"data-svelte-h":!0}),b(q)!=="svelte-1cnm7jd"&&(q.textContent=Te),Hl=y(l),T(D.$$.fragment,l),Sl=y(l),L=j(l,"P",{"data-svelte-h":!0}),b(L)!=="svelte-vro956"&&(L.textContent=we),ql=y(l),T(K.$$.fragment,l),Dl=y(l),T(P.$$.fragment,l),Ll=y(l),O=j(l,"P",{"data-svelte-h":!0}),b(O)!=="svelte-1w07l5t"&&(O.textContent=ue),Kl=y(l),T(ll.$$.fragment,l),Pl=y(l),T(sl.$$.fragment,l),Js=y(l),T(al.$$.fragment,l),pl=y(l),yl=j(l,"P",{"data-svelte-h":!0}),b(yl)!=="svelte-zsplm1"&&(yl.textContent=Bl),Ol=y(l),T(el.$$.fragment,l),os=y(l),T(rl.$$.fragment,l),cl=y(l),il=j(l,"P",{"data-svelte-h":!0}),b(il)!=="svelte-1cwvi0h"&&(il.innerHTML=Xl),Us=y(l),T(Jl.$$.fragment,l),ol=y(l),T(Ul.$$.fragment,l),ml=y(l),Tl=j(l,"P",{"data-svelte-h":!0}),b(Tl)!=="svelte-ht5gll"&&(Tl.textContent=Rl),ms=y(l),wl=j(l,"P",{"data-svelte-h":!0}),b(wl)!=="svelte-1voe59b"&&(wl.innerHTML=Ql),ls=y(l),E=j(l,"DIV",{class:!0,"data-svelte-h":!0}),b(E)!=="svelte-1mf8pz"&&(E.innerHTML=de),ss=y(l),tl=j(l,"P",{"data-svelte-h":!0}),b(tl)!=="svelte-1qok9pu"&&(tl.textContent=Ts),ul=y(l),T(dl.$$.fragment,l),s=y(l),T(U.$$.fragment,l),be=y(l),ws=j(l,"P",{"data-svelte-h":!0}),b(ws)!=="svelte-13p5c1a"&&(ws.innerHTML=mt),fe=y(l),us=j(l,"P",{"data-svelte-h":!0}),b(us)!=="svelte-4dr9zl"&&(us.innerHTML=Tt),Ce=y(l),T(ds.$$.fragment,l),he=y(l),T(js.$$.fragment,l),Ie=y(l),bs=j(l,"P",{"data-svelte-h":!0}),b(bs)!=="svelte-3miggq"&&(bs.textContent=wt),_e=y(l),fs=j(l,"P",{"data-svelte-h":!0}),b(fs)!=="svelte-1icb7xl"&&(fs.textContent=ut),ke=y(l),T(ts.$$.fragment,l),Ze=y(l),T(Cs.$$.fragment,l),$e=y(l),hs=j(l,"P",{"data-svelte-h":!0}),b(hs)!=="svelte-esxj5c"&&(hs.innerHTML=dt),Ge=y(l),bl.l(l),te=y(l),Is=j(l,"P",{"data-svelte-h":!0}),b(Is)!=="svelte-19no68y"&&(Is.innerHTML=jt),ge=y(l),_s=j(l,"P",{"data-svelte-h":!0}),b(_s)!=="svelte-1vi62i3"&&(_s.innerHTML=bt),Be=y(l),Cl.l(l),ae=y(l),ks=j(l,"P",{"data-svelte-h":!0}),b(ks)!=="svelte-ccdb1w"&&(ks.textContent=ft),Xe=y(l),T(Zs.$$.fragment,l),Re=y(l),Il.l(l),ne=y(l),$s=j(l,"P",{"data-svelte-h":!0}),b($s)!=="svelte-pw44us"&&($s.textContent=Ct),Qe=y(l),Q&&Q.l(l),Me=y(l),T(as.$$.fragment,l),We=y(l),Gs=j(l,"P",{"data-svelte-h":!0}),b(Gs)!=="svelte-147y40m"&&(Gs.textContent=ht),Ve=y(l),T(gs.$$.fragment,l),Ee=y(l),Bs=j(l,"P",{"data-svelte-h":!0}),b(Bs)!=="svelte-kumw5c"&&(Bs.textContent=It),ze=y(l),Xs=j(l,"P",{"data-svelte-h":!0}),b(Xs)!=="svelte-19efumu"&&(Xs.textContent=_t),Ae=y(l),T(Rs.$$.fragment,l),ve=y(l),kl.l(l),pe=y(l),T(ns.$$.fragment,l),Ne=y(l),T(Ms.$$.fragment,l),xe=y(l),T(Qs.$$.fragment,l),Fe=y(l),Ws=j(l,"P",{"data-svelte-h":!0}),b(Ws)!=="svelte-1nsc380"&&(Ws.innerHTML=kt),Ye=y(l),$l.l(l),ye=y(l),Vs=j(l,"P",{"data-svelte-h":!0}),b(Vs)!=="svelte-1qna6q5"&&(Vs.textContent=Zt),He=y(l),T(Es.$$.fragment,l),Se=y(l),T(zs.$$.fragment,l),qe=y(l),As=j(l,"P",{"data-svelte-h":!0}),b(As)!=="svelte-fxo5s1"&&(As.innerHTML=$t),De=y(l),T(vs.$$.fragment,l),Le=y(l),T(Ns.$$.fragment,l),Ke=y(l),xs=j(l,"P",{"data-svelte-h":!0}),b(xs)!=="svelte-1lches0"&&(xs.innerHTML=Gt),Pe=y(l),T(Fs.$$.fragment,l),Oe=y(l),T(Ys.$$.fragment,l),lt=y(l),Hs=j(l,"P",{"data-svelte-h":!0}),b(Hs)!=="svelte-6udiea"&&(Hs.innerHTML=gt),st=y(l),T(Ss.$$.fragment,l),et=y(l),T(qs.$$.fragment,l),tt=y(l),es.l(l),re=y(l),W&&W.l(l),ce=y(l),T(Ds.$$.fragment,l),at=y(l),je=j(l,"P",{}),ot(je).forEach(t),this.h()},h(){Wl(e,"name","hf:doc:metadata"),Wl(e,"content",Qa),ea(X.src,oe="https://course-demos-codeparrot-ds.hf.space")||Wl(X,"src",oe),Wl(X,"frameborder","0"),Wl(X,"height","300"),Wl(X,"title","Gradio app"),Wl(X,"class","block dark:hidden container p-0 flex-grow space-iframe"),Wl(X,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),Wl(X,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),Wl(E,"class","flex justify-center")},m(l,M){pa(document.head,e),a(l,r,M),a(l,n,M),a(l,J,M),w(_,l,M),a(l,f,M),w(G,l,M),a(l,$,M),Ls[Z].m(l,M),a(l,C,M),a(l,g,M),a(l,V,M),a(l,R,M),a(l,B,M),w(A,l,M),a(l,Vl,M),a(l,v,M),a(l,El,M),a(l,X,M),a(l,zl,M),a(l,N,M),a(l,Al,M),w(x,l,M),a(l,vl,M),a(l,F,M),a(l,Nl,M),a(l,Y,M),a(l,nl,M),w(Gl,l,M),a(l,xl,M),a(l,H,M),a(l,Ml,M),w(gl,l,M),a(l,Fl,M),w(S,l,M),a(l,Yl,M),a(l,q,M),a(l,Hl,M),w(D,l,M),a(l,Sl,M),a(l,L,M),a(l,ql,M),w(K,l,M),a(l,Dl,M),w(P,l,M),a(l,Ll,M),a(l,O,M),a(l,Kl,M),w(ll,l,M),a(l,Pl,M),w(sl,l,M),a(l,Js,M),w(al,l,M),a(l,pl,M),a(l,yl,M),a(l,Ol,M),w(el,l,M),a(l,os,M),w(rl,l,M),a(l,cl,M),a(l,il,M),a(l,Us,M),w(Jl,l,M),a(l,ol,M),w(Ul,l,M),a(l,ml,M),a(l,Tl,M),a(l,ms,M),a(l,wl,M),a(l,ls,M),a(l,E,M),a(l,ss,M),a(l,tl,M),a(l,ul,M),w(dl,l,M),a(l,s,M),w(U,l,M),a(l,be,M),a(l,ws,M),a(l,fe,M),a(l,us,M),a(l,Ce,M),w(ds,l,M),a(l,he,M),w(js,l,M),a(l,Ie,M),a(l,bs,M),a(l,_e,M),a(l,fs,M),a(l,ke,M),w(ts,l,M),a(l,Ze,M),w(Cs,l,M),a(l,$e,M),a(l,hs,M),a(l,Ge,M),Ks[jl].m(l,M),a(l,te,M),a(l,Is,M),a(l,ge,M),a(l,_s,M),a(l,Be,M),Ps[fl].m(l,M),a(l,ae,M),a(l,ks,M),a(l,Xe,M),w(Zs,l,M),a(l,Re,M),Os[hl].m(l,M),a(l,ne,M),a(l,$s,M),a(l,Qe,M),Q&&Q.m(l,M),a(l,Me,M),w(as,l,M),a(l,We,M),a(l,Gs,M),a(l,Ve,M),w(gs,l,M),a(l,Ee,M),a(l,Bs,M),a(l,ze,M),a(l,Xs,M),a(l,Ae,M),w(Rs,l,M),a(l,ve,M),le[_l].m(l,M),a(l,pe,M),w(ns,l,M),a(l,Ne,M),w(Ms,l,M),a(l,xe,M),w(Qs,l,M),a(l,Fe,M),a(l,Ws,M),a(l,Ye,M),se[Zl].m(l,M),a(l,ye,M),a(l,Vs,M),a(l,He,M),w(Es,l,M),a(l,Se,M),w(zs,l,M),a(l,qe,M),a(l,As,M),a(l,De,M),w(vs,l,M),a(l,Le,M),w(Ns,l,M),a(l,Ke,M),a(l,xs,M),a(l,Pe,M),w(Fs,l,M),a(l,Oe,M),w(Ys,l,M),a(l,lt,M),a(l,Hs,M),a(l,st,M),w(Ss,l,M),a(l,et,M),w(qs,l,M),a(l,tt,M),es.m(l,M),a(l,re,M),W&&W.m(l,M),a(l,ce,M),w(Ds,l,M),a(l,at,M),a(l,je,M),nt=!0},p(l,[M]){const Yt={};M&1&&(Yt.fw=l[0]),_.$set(Yt);let pt=Z;Z=Xt(l),Z!==pt&&(ys(),i(Ls[pt],1,1,()=>{Ls[pt]=null}),ps(),o=Ls[Z],o||(o=Ls[Z]=Bt[Z](l),o.c()),c(o,1),o.m(C.parentNode,C));const Ht={};M&2&&(Ht.$$scope={dirty:M,ctx:l}),al.$set(Ht);const St={};M&2&&(St.$$scope={dirty:M,ctx:l}),ts.$set(St);let yt=jl;jl=Qt(l),jl!==yt&&(ys(),i(Ks[yt],1,1,()=>{Ks[yt]=null}),ps(),bl=Ks[jl],bl||(bl=Ks[jl]=Rt[jl](l),bl.c()),c(bl,1),bl.m(te.parentNode,te));let rt=fl;fl=Vt(l),fl!==rt&&(ys(),i(Ps[rt],1,1,()=>{Ps[rt]=null}),ps(),Cl=Ps[fl],Cl||(Cl=Ps[fl]=Wt[fl](l),Cl.c()),c(Cl,1),Cl.m(ae.parentNode,ae));let ct=hl;hl=zt(l),hl!==ct&&(ys(),i(Os[ct],1,1,()=>{Os[ct]=null}),ps(),Il=Os[hl],Il||(Il=Os[hl]=Et[hl](l),Il.c()),c(Il,1),Il.m(ne.parentNode,ne)),l[0]==="tf"?Q?M&1&&c(Q,1):(Q=Pt(),Q.c(),c(Q,1),Q.m(Me.parentNode,Me)):Q&&(ys(),i(Q,1,1,()=>{Q=null}),ps());const qt={};M&2&&(qt.$$scope={dirty:M,ctx:l}),as.$set(qt);let it=_l;_l=vt(l),_l!==it&&(ys(),i(le[it],1,1,()=>{le[it]=null}),ps(),kl=le[_l],kl||(kl=le[_l]=At[_l](l),kl.c()),c(kl,1),kl.m(pe.parentNode,pe));const Dt={};M&2&&(Dt.$$scope={dirty:M,ctx:l}),ns.$set(Dt);const Lt={};M&3&&(Lt.$$scope={dirty:M,ctx:l}),Ms.$set(Lt);let Jt=Zl;Zl=xt(l),Zl!==Jt&&(ys(),i(se[Jt],1,1,()=>{se[Jt]=null}),ps(),$l=se[Zl],$l||($l=se[Zl]=Nt[Zl](l),$l.c()),c($l,1),$l.m(ye.parentNode,ye)),Mt!==(Mt=Ft(l))&&(es.d(1),es=Mt(l),es&&(es.c(),es.m(re.parentNode,re))),l[0]==="pt"?W?M&1&&c(W,1):(W=Ot(l),W.c(),c(W,1),W.m(ce.parentNode,ce)):W&&(ys(),i(W,1,1,()=>{W=null}),ps())},i(l){nt||(c(_.$$.fragment,l),c(G.$$.fragment,l),c(o),c(A.$$.fragment,l),c(x.$$.fragment,l),c(Gl.$$.fragment,l),c(gl.$$.fragment,l),c(S.$$.fragment,l),c(D.$$.fragment,l),c(K.$$.fragment,l),c(P.$$.fragment,l),c(ll.$$.fragment,l),c(sl.$$.fragment,l),c(al.$$.fragment,l),c(el.$$.fragment,l),c(rl.$$.fragment,l),c(Jl.$$.fragment,l),c(Ul.$$.fragment,l),c(dl.$$.fragment,l),c(U.$$.fragment,l),c(ds.$$.fragment,l),c(js.$$.fragment,l),c(ts.$$.fragment,l),c(Cs.$$.fragment,l),c(bl),c(Cl),c(Zs.$$.fragment,l),c(Il),c(Q),c(as.$$.fragment,l),c(gs.$$.fragment,l),c(Rs.$$.fragment,l),c(kl),c(ns.$$.fragment,l),c(Ms.$$.fragment,l),c(Qs.$$.fragment,l),c($l),c(Es.$$.fragment,l),c(zs.$$.fragment,l),c(vs.$$.fragment,l),c(Ns.$$.fragment,l),c(Fs.$$.fragment,l),c(Ys.$$.fragment,l),c(Ss.$$.fragment,l),c(qs.$$.fragment,l),c(W),c(Ds.$$.fragment,l),nt=!0)},o(l){i(_.$$.fragment,l),i(G.$$.fragment,l),i(o),i(A.$$.fragment,l),i(x.$$.fragment,l),i(Gl.$$.fragment,l),i(gl.$$.fragment,l),i(S.$$.fragment,l),i(D.$$.fragment,l),i(K.$$.fragment,l),i(P.$$.fragment,l),i(ll.$$.fragment,l),i(sl.$$.fragment,l),i(al.$$.fragment,l),i(el.$$.fragment,l),i(rl.$$.fragment,l),i(Jl.$$.fragment,l),i(Ul.$$.fragment,l),i(dl.$$.fragment,l),i(U.$$.fragment,l),i(ds.$$.fragment,l),i(js.$$.fragment,l),i(ts.$$.fragment,l),i(Cs.$$.fragment,l),i(bl),i(Cl),i(Zs.$$.fragment,l),i(Il),i(Q),i(as.$$.fragment,l),i(gs.$$.fragment,l),i(Rs.$$.fragment,l),i(kl),i(ns.$$.fragment,l),i(Ms.$$.fragment,l),i(Qs.$$.fragment,l),i($l),i(Es.$$.fragment,l),i(zs.$$.fragment,l),i(vs.$$.fragment,l),i(Ns.$$.fragment,l),i(Fs.$$.fragment,l),i(Ys.$$.fragment,l),i(Ss.$$.fragment,l),i(qs.$$.fragment,l),i(W),i(Ds.$$.fragment,l),nt=!1},d(l){l&&(t(r),t(n),t(J),t(f),t($),t(C),t(g),t(V),t(R),t(B),t(Vl),t(v),t(El),t(X),t(zl),t(N),t(Al),t(vl),t(F),t(Nl),t(Y),t(nl),t(xl),t(H),t(Ml),t(Fl),t(Yl),t(q),t(Hl),t(Sl),t(L),t(ql),t(Dl),t(Ll),t(O),t(Kl),t(Pl),t(Js),t(pl),t(yl),t(Ol),t(os),t(cl),t(il),t(Us),t(ol),t(ml),t(Tl),t(ms),t(wl),t(ls),t(E),t(ss),t(tl),t(ul),t(s),t(be),t(ws),t(fe),t(us),t(Ce),t(he),t(Ie),t(bs),t(_e),t(fs),t(ke),t(Ze),t($e),t(hs),t(Ge),t(te),t(Is),t(ge),t(_s),t(Be),t(ae),t(ks),t(Xe),t(Re),t(ne),t($s),t(Qe),t(Me),t(We),t(Gs),t(Ve),t(Ee),t(Bs),t(ze),t(Xs),t(Ae),t(ve),t(pe),t(Ne),t(xe),t(Fe),t(Ws),t(Ye),t(ye),t(Vs),t(He),t(Se),t(qe),t(As),t(De),t(Le),t(Ke),t(xs),t(Pe),t(Oe),t(lt),t(Hs),t(st),t(et),t(tt),t(re),t(ce),t(at),t(je)),t(e),u(_,l),u(G,l),Ls[Z].d(l),u(A,l),u(x,l),u(Gl,l),u(gl,l),u(S,l),u(D,l),u(K,l),u(P,l),u(ll,l),u(sl,l),u(al,l),u(el,l),u(rl,l),u(Jl,l),u(Ul,l),u(dl,l),u(U,l),u(ds,l),u(js,l),u(ts,l),u(Cs,l),Ks[jl].d(l),Ps[fl].d(l),u(Zs,l),Os[hl].d(l),Q&&Q.d(l),u(as,l),u(gs,l),u(Rs,l),le[_l].d(l),u(ns,l),u(Ms,l),u(Qs,l),se[Zl].d(l),u(Es,l),u(zs,l),u(vs,l),u(Ns,l),u(Fs,l),u(Ys,l),u(Ss,l),u(qs,l),es.d(l),W&&W.d(l),u(Ds,l)}}}const Qa='{"title":"从头开始训练因果语言模型","local":"从头开始训练因果语言模型","sections":[{"title":"收集数据","local":"收集数据","sections":[],"depth":2},{"title":"准备数据集","local":"准备数据集","sections":[],"depth":2},{"title":"初始化新模型","local":"初始化新模型","sections":[],"depth":2},{"title":"使用管道生成代码","local":"使用管道生成代码","sections":[],"depth":2},{"title":"用 🤗 Accelerate 训练","local":"用 🤗 Accelerate 训练","sections":[],"depth":2}],"depth":1}';function Wa(I,e,r){let n="pt";return ta(()=>{const J=new URLSearchParams(window.location.search);r(0,n=J.get("fw")||"pt")}),[n]}class Ya extends aa{constructor(e){super(),na(this,e,Wa,Ra,sa,{})}}export{Ya as component};
