import{s as q,n as F,o as G}from"../chunks/scheduler.37c15a92.js";import{S as O,i as R,g as c,s as i,r as T,A as W,h as $,f as a,c as l,j as B,u as y,x as z,k as N,y as D,a as n,v as H,d as L,t as M,w as j}from"../chunks/index.2bf4358c.js";import{C as I}from"../chunks/CourseFloatingBanner.06521e8b.js";import{H as J,E as K}from"../chunks/EditOnGithub.8916dc7c.js";function Q(S){let s,d,h,g,m,x,r,C,o,U="完成这一章，辛苦了！",v,p,k="在深入研究标记器之后，您应该：",P,u,A="<li>能够使用旧的标记器作为模板来训练新的标记器</li> <li>了解如何使用偏移量将标记的位置映射到其原始文本范围</li> <li>了解 BPE、WordPiece 和 Unigram 之间的区别</li> <li>能够混合和匹配 🤗 Tokenizers 库提供的块来构建您自己的标记器</li> <li>能够在 🤗 Transformers 库中使用该标记器</li>",b,f,w,_,E;return m=new J({props:{title:"标记器，回顾！",local:"标记器，回顾！",headingTag:"h1"}}),r=new I({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),f=new K({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-CN/chapter6/9.mdx"}}),{c(){s=c("meta"),d=i(),h=c("p"),g=i(),T(m.$$.fragment),x=i(),T(r.$$.fragment),C=i(),o=c("p"),o.textContent=U,v=i(),p=c("p"),p.textContent=k,P=i(),u=c("ul"),u.innerHTML=A,b=i(),T(f.$$.fragment),w=i(),_=c("p"),this.h()},l(t){const e=W("svelte-u9bgzb",document.head);s=$(e,"META",{name:!0,content:!0}),e.forEach(a),d=l(t),h=$(t,"P",{}),B(h).forEach(a),g=l(t),y(m.$$.fragment,t),x=l(t),y(r.$$.fragment,t),C=l(t),o=$(t,"P",{"data-svelte-h":!0}),z(o)!=="svelte-cpcc87"&&(o.textContent=U),v=l(t),p=$(t,"P",{"data-svelte-h":!0}),z(p)!=="svelte-1ajcmaj"&&(p.textContent=k),P=l(t),u=$(t,"UL",{"data-svelte-h":!0}),z(u)!=="svelte-13nu9mu"&&(u.innerHTML=A),b=l(t),y(f.$$.fragment,t),w=l(t),_=$(t,"P",{}),B(_).forEach(a),this.h()},h(){N(s,"name","hf:doc:metadata"),N(s,"content",V)},m(t,e){D(document.head,s),n(t,d,e),n(t,h,e),n(t,g,e),H(m,t,e),n(t,x,e),H(r,t,e),n(t,C,e),n(t,o,e),n(t,v,e),n(t,p,e),n(t,P,e),n(t,u,e),n(t,b,e),H(f,t,e),n(t,w,e),n(t,_,e),E=!0},p:F,i(t){E||(L(m.$$.fragment,t),L(r.$$.fragment,t),L(f.$$.fragment,t),E=!0)},o(t){M(m.$$.fragment,t),M(r.$$.fragment,t),M(f.$$.fragment,t),E=!1},d(t){t&&(a(d),a(h),a(g),a(x),a(C),a(o),a(v),a(p),a(P),a(u),a(b),a(w),a(_)),a(s),j(m,t),j(r,t),j(f,t)}}}const V='{"title":"标记器，回顾！","local":"标记器，回顾！","sections":[],"depth":1}';function X(S){return G(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class at extends O{constructor(s){super(),R(this,s,X,Q,q,{})}}export{at as component};
