LICENSE
MANIFEST.in
README.md
pyproject.toml
setup.py
codegen/gen_dict.py
codegen/gen_precomputed.py
data/cl100k_base.tiktoken.gz
data/o200k_base.tiktoken.gz
data/precomputed_cl100k.bin.gz
data/precomputed_o200k.bin.gz
include/ac_bpe.h
include/appendable_encoder.h
include/bitfield.h
include/bpe.h
include/c11_threads.h
include/fnv_hash.h
include/interval_encoding.h
include/lru_cache.h
include/prependable_encoder.h
include/pretok.h
include/pretok_ucd.h
include/threadpool.h
include/tokenizer.h
python/c_bpe/__init__.py
python/c_bpe/openai.py
python/c_bpe/py.typed
python/c_bpe.egg-info/PKG-INFO
python/c_bpe.egg-info/SOURCES.txt
python/c_bpe.egg-info/dependency_links.txt
python/c_bpe.egg-info/requires.txt
python/c_bpe.egg-info/top_level.txt
src/ac_bpe.c
src/ac_bpe_old.c
src/appendable_encoder.c
src/bpe_core.c
src/dict_cl100k.h
src/dict_o200k.h
src/interval_encoding.c
src/parallel.c
src/prependable_encoder.c
src/pretok_cl100k.c
src/pretok_o200k.c
src/pymodule.c
src/threadpool.c
src/tokenizer.c
tests/conftest.py
tests/test.py
tests/test_basic.py
tests/test_benchmarks.py
tests/test_import.py
third_party/pcre2/config.h
third_party/pcre2/pcre2.h
third_party/pcre2/src/pcre2_internal.h
third_party/pcre2/src/pcre2_intmodedep.h
third_party/pcre2/src/pcre2_tables.c
third_party/pcre2/src/pcre2_ucd.c
third_party/pcre2/src/pcre2_ucp.h
third_party/pcre2/src/pcre2_ucptables.c
third_party/pcre2/src/pcre2_util.h