feat: add newly trained tokenizer (final one)
Browse files- token_bytes.pt +1 -1
- tokenizer.pkl +2 -2
token_bytes.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 263721
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e9471d41c2bafb5dbdca80858bfaf9fe5e17ab5f9b8740cf40f099a9d54413c0
|
| 3 |
size 263721
|
tokenizer.pkl
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a78a49fc0fe02a6988df6b3469eb284503e2de52dc99d0abee9389b4f4677eb7
|
| 3 |
+
size 888839
|