import six
import os
import numpy as np
import paddle
from psutil import cpu_count
from paddlenlp.transformers import AutoTokenizer
import json
import time


model_name_or_path="ernie-3.0-medium-zh"
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_faster=True)

data = [[" "]]
ret = tokenizer(data,
                        max_length=512,
                        padding=True,
                        truncation=True,
                        is_split_into_words=True)
print(ret)

### error：
"""
terminate called after throwing an instance of 'std::logic_error'
  what():  The split of PreTokenizedString is empty, please call PreTokenizedString::Tokenize first before transform to Encoding.
Aborted
"""