Source code for flambe.tokenizer.char
from typing import List
from flambe.tokenizer import Tokenizer
[docs]class CharTokenizer(Tokenizer):
"""Implement a character level tokenizer."""
[docs] def tokenize(self, example: str) -> List[str]:
"""Tokenize an input example.
Parameters
----------
example : str
The input example, as a string
Returns
-------
List[str]
The output charachter tokens, as a list of strings
"""
return list(example)