add character glm

vocab/__init__.py

@@ -19,32 +19,37 @@ tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.c
     - 特征
       - 词典：有##开头的token，表示subword
     - 示例：
-  - google/sentencepiece: 
+  - bpe-google/sentencepiece: 
     - 特征：
       - 训练:  
-      - 文件: *.sp_model  或 *.model  (可选文件 .vocab，)
+      - 文件: *.sp_model  或 *.model  (可选文件 .vocab，) spm简称
       - 实现: 
+        - 依赖: protobuf
         - 训练: `import sentencepiece as spm; spm.SentencePieceTrainer.train` 或 `spm_train`
         - 加载: `import sentencepiece as spm; spm.SentencePieceProcessor().Load(vocab_file)`
         - 方法: 是SentencePieceProcessor类型，sp_model.id_to_piece，有tokenizer.json tokenizer.model，
+        - 分词:
+          - pre_tokenizers.ByteLevel(add_prefix_space=True, use_regex=False)
       - 词典:  词典字符有 ▁  (U+2581) ，表示空格或句首。
     - 示例：google-t5, llama，baichuan, orion, 
   - icetk： sentencepiece的分支，支持image_tokenizer
     - glm, chatglm1, chatglm2
   - openai/tiktoken
-  - hf_tokenizer
+  - bpe-hf_tokenizer
+    - ss
     - 特征：
       - 文件: tokenizer.json(包含后两个文件的内容), merges.txt, vocab.json
         - added_tokens 在vocab中不一定存在。
       - 实现:
-        - 训练:
+        - 训练: `from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer`
         - 加载:
-        - 方法: 
+        - 方法: .model.from_file  .model.save   .model.token_to_id  .model.tokenize
       - .model 是 tokenizer.models.BPE 类型
       - 词典有 Ġ  "\u0120" 开头
-      - .model.from_file  .model.save   .model.token_to_id  .model.tokenize
+      - 优势
+      - 
     - 示例：gpt2, gpt_neox_20b, moss, bloom, qwen2
-    - 构造词典：
+    - 优势：相对sentence piece，hf_tokenizer支持pre-tokenization的正则表达式，对tab和换行支持更好 ()
       - ss
   - tiktoken
     - 特征：空格就是空格，
@@ -65,71 +70,72 @@ uniq_tokenizers = [
     ""
 ]
 
-# TODO: alias/abbr, hf_path, tokenizer_class, comments,
+# TODO: alias/abbr, hf_path, tokenizer_class/type, comments,
 all_tokenizers = [
     ##### bert 系列
-    ("bert_base_cased", "", ""),
-    ("bert_base_uncased","",),
-    ("bert_base_chinese",),
-    ("roberta_chinese_clue",),
+    ("bert_base_cased", "", "bert"),
+    ("bert_base_uncased", "", "bert"),
+    ("bert_base_chinese", "", "bert"),
+    ("roberta_chinese_clue", "", "bert"),
     ("kplug",),
     ("gpt2_chinese",),
 
     ##### GPT2Tokenizer
-    ("gpt2",),   #
-    ("moss",),
-    ("bloom",),
+    ("gpt2", "", "GPT2Tokenizer",),  #
+    ("moss", "", "GPT2Tokenizer",),
+    ("bloom", "", "GPT2Tokenizer",),
     # ("bloomz_6b4_zh",
     # ("belle_7b_2m",   # 模型和词典都基于bloom
     #
-    ("gpt_nexo_20b",),      # 5万
-    ("qwen1_5_14b_chat",),  # 15万，速度有点慢
-    ("starchat_alpha",),
+    ("gpt_nexo_20b", "", "GPT2Tokenizer",),  # 5万
+    ("qwen1_5_14b_chat", "", "GPT2Tokenizer",),  # 15万，速度有点慢
+    ("starchat_alpha", "", "GPT2Tokenizer",),
 
     ####### google/sentencepiece tokenizer:
     # T5 llama internlm
-    ("t5_small",),
-    ("t5_base",),
-    ("t5_large",),
-    ("chatyuan_large_v2",),
-    ("prompt_clue",),
-
-    ("llama",),  # '中文单字': 700, '中文多字': 0
-    ("llama2",),
-    ("chinese_llama",),  #
-    ("chinese_llama2",),  #
+    ("t5_small", "", "sentencepiece"),
+    ("t5_base", "", "sentencepiece"),
+    ("t5_large", "", "sentencepiece"),
+    ("chatyuan_large_v2", "", "sentencepiece"),
+    ("prompt_clue", "", "sentencepiece"),
+
+    ("llama", "", "sentencepiece"),  # '中文单字': 700, '中文多字': 0
+    ("llama2", "", "sentencepiece"),
+    ("chinese_llama", "", "sentencepiece"),  #
+    ("chinese_llama2", "", "sentencepiece"),  #
     # ("chinese_alpaca_lora_7b",  # 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。
     # ("belle_llama_ext_7b",
     # ("alpaca_7b",
-    ("baichuan",),
-    ("baichuan2",),
-    ("internlm_chat_7b",),
-    ("internlm2_chat_7b",),
-    ("internlm2_math_7b",),
-    ("internlm_xcomposer_7b",),
-    ("falcon_7b",),
-    ("falcon_180b",),
+    ("baichuan", "", "sentencepiece"),
+    ("baichuan2", "", "sentencepiece"),
+    ("internlm_chat_7b", "", "sentencepiece"),
+    ("internlm2_chat_7b", "", "sentencepiece"),
+    ("internlm2_math_7b", "", "sentencepiece"),
+    ("internlm_xcomposer_7b", "", "sentencepiece"),
+    ("falcon_7b", "", "sentencepiece"),
+    ("falcon_180b", "", "sentencepiece"),
+    ("skywork_13b_base",),
+    ("skywork_13b_math",),
     # "goat",
 
     # ##### glm系列
     # "glm_chinese",),
-    ("chatglm_6b",),
-    ("chatglm2_6b",),
-    ("chatglm3_6b",),
-
+    ("chatglm_6b", "", "sentencepiece"),
+    ("chatglm2_6b", "", "sentencepiece"),
+    ("chatglm3_6b", "", "sentencepiece"),
+    ("character_glm_6b", "", "sentencepiece"),
 
     # tiktoken 系列
-    ("qwen_1_8b_chat",),
-    ("qwen_7b_chat",),
-    ("qwen_72b_chat",),
-    ("text_davinci_003",),
-    ("code_davinci_002",),
-    ("gpt_35_turbo",),
-    ("gpt_4",),
+    ("qwen_1_8b_chat", "", "tiktoken"),
+    ("qwen_7b_chat", "", "tiktoken"),
+    ("qwen_72b_chat", "", "tiktoken"),
+    ("text_davinci_003", "", "tiktoken"),
+    ("code_davinci_002", "", "tiktoken"),
+    ("gpt_35_turbo", "", "tiktoken"),
+    ("gpt_4", "", "tiktoken"),
 
     # 未分类
-    ("skywork_13b_base",),
-    ("skywork_13b_math",),
+
     ("mistral_7b",),
     ("mixtral_8_7b",),
 
@@ -205,15 +211,21 @@ class TokenizerType(Enum):
 
 
 class TokenizerImpl(Enum):
+
     """
+    https://github.com/google/sentencepiece，支持 sentencepiece(BPE,unigram,char,word), wordpiece,
+    spm_train --model_type unigram/bpe/char/word
     """
-    SentencePiece = auto()  #
+    SentencePiece = auto()
 
     # https://github.com/huggingface/transformers/blob/v4.30.2/src/transformers/models/gpt2/tokenization_gpt2.py#L104
     # 构造词典：
-    #
-    GPT2Tokenizer = auto()
-    BertTokenizer = auto()  #
+    # GPT2Tokenizer = auto()
+    # BertTokenizer = auto()  #
+
+    """
+    """
+    HFTokenizer = auto()  # https://github.com/huggingface/tokenizers, 支持
 
 
 def load_tokener(model_name):

vocab/character_glm_6b/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("thu-coai/CharacterGLM-6B", trust_remote_code=True)