{
  "_id": "6a5b0da59baadeef8ab4aada",
  "Package": "tok",
  "Title": "Fast Text Tokenization",
  "Version": "0.2.2.9000",
  "Authors@R": "c(\nperson(\"Tomasz\", \"Kalinowski\", , \"tomasz@posit.co\", c(\"ctb\", \"cre\")),\nperson(\"Daniel\", \"Falbel\", , \"dfalbel@gmail.com\", c(\"aut\")),\nperson(\"Regouby\", \"Christophe\", , \"christophe.regouby@free.fr\", c(\"ctb\")),\nperson(family = \"Posit\", role = c(\"cph\"))\n)",
  "Description": "Interfaces with the 'Hugging Face' tokenizers library to\nprovide implementations of today's most used tokenizers such as\nthe 'Byte-Pair Encoding' algorithm\n<https://huggingface.co/docs/tokenizers/index>. It's extremely\nfast for both training new vocabularies and tokenizing texts.",
  "License": "MIT + file LICENSE",
  "SystemRequirements": "Cargo (Rust's package manager), rustc >= 1.77.2",
  "Encoding": "UTF-8",
  "Roxygen": "list(markdown = TRUE)",
  "Config/testthat/edition": "3",
  "URL": "https://github.com/mlverse/tok",
  "BugReports": "https://github.com/mlverse/tok/issues",
  "Config/rextendr/version": "0.5.0",
  "Config/roxygen2/version": "8.0.0",
  "Config/pak/sysreqs": "libclang-dev",
  "Repository": "https://mlverse.r-universe.dev",
  "Date/Publication": "2026-05-19 13:09:10 UTC",
  "RemoteUrl": "https://github.com/mlverse/tok",
  "RemoteRef": "HEAD",
  "RemoteSha": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
  "NeedsCompilation": "yes",
  "Packaged": {
    "Date": "2026-07-18 05:13:28 UTC",
    "User": "root"
  },
  "Author": "Tomasz Kalinowski [ctb, cre],\nDaniel Falbel [aut],\nRegouby Christophe [ctb],\nPosit [cph]",
  "Maintainer": "Tomasz Kalinowski <tomasz@posit.co>",
  "_user": "mlverse",
  "_type": "src",
  "_file": "tok_0.2.2.9000.tar.gz",
  "_fileid": "https://r2.ropensci.org/08ab8bcdb8b5f9383496cc9a066fdb163a867b47d1f29bbd9230c3f19d3decf4",
  "_filesize": 11765258,
  "_sha256": "08ab8bcdb8b5f9383496cc9a066fdb163a867b47d1f29bbd9230c3f19d3decf4",
  "_expires": "2026-10-26T05:22:43.000Z",
  "_created": "2026-07-18T05:13:28.000Z",
  "_published": "2026-07-18T05:22:45.768Z",
  "_jobs": [
    {
      "job": 88047281580,
      "time": 272,
      "config": "linux-devel-arm64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "8425812657"
    },
    {
      "job": 88047281577,
      "time": 278,
      "config": "linux-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "8425813283"
    },
    {
      "job": 88047281584,
      "time": 264,
      "config": "linux-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8425811409"
    },
    {
      "job": 88047281590,
      "time": 256,
      "config": "linux-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8425809826"
    },
    {
      "job": 88047281539,
      "time": 180,
      "config": "macos-oldrel-arm64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "8425796802"
    },
    {
      "job": 88047281561,
      "time": 418,
      "config": "macos-oldrel-x86_64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "8425835373"
    },
    {
      "job": 88047281554,
      "time": 174,
      "config": "macos-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8425795631"
    },
    {
      "job": 88047281578,
      "time": 311,
      "config": "macos-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8425818386"
    },
    {
      "job": 88046822115,
      "time": 312,
      "config": "source",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8425766008"
    },
    {
      "job": 88047281537,
      "time": 216,
      "config": "wasm-release",
      "r": "4.6.0",
      "check": "FAIL",
      "artifact": ""
    },
    {
      "job": 88047281550,
      "time": 316,
      "config": "windows-devel",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "8425819068"
    },
    {
      "job": 88047281536,
      "time": 330,
      "config": "windows-oldrel",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "8425821667"
    },
    {
      "job": 88047281535,
      "time": 298,
      "config": "windows-release",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8425816040"
    }
  ],
  "_host": "GitHub-Actions",
  "_buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448",
  "_status": "success",
  "_upstream": "https://github.com/mlverse/tok",
  "_commit": {
    "id": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
    "author": "Tomasz Kalinowski <kalinowskit@gmail.com>",
    "committer": "Tomasz Kalinowski <kalinowskit@gmail.com>",
    "message": "update `cran-comments.md`\n",
    "time": 1779196150
  },
  "_maintainer": {
    "name": "Tomasz Kalinowski",
    "email": "tomasz@posit.co",
    "login": "t-kalinowski",
    "mastodon": "@t_kalinowski@fosstodon.org",
    "bluesky": "@t-kalinowski.bsky.social",
    "description": "",
    "uuid": 8462255
  },
  "_distro": "resolute",
  "_registered": true,
  "_dependencies": [
    {
      "package": "R",
      "version": ">= 4.2.0",
      "role": "Depends"
    },
    {
      "package": "R6",
      "role": "Imports"
    },
    {
      "package": "cli",
      "role": "Imports"
    },
    {
      "package": "rmarkdown",
      "role": "Suggests"
    },
    {
      "package": "testthat",
      "version": ">= 3.0.0",
      "role": "Suggests"
    },
    {
      "package": "hfhub",
      "version": ">= 0.1.1",
      "role": "Suggests"
    },
    {
      "package": "withr",
      "role": "Suggests"
    }
  ],
  "_owner": "mlverse",
  "_selfowned": true,
  "_usedby": 1,
  "_updates": [
    {
      "week": "2025-35",
      "n": 7
    },
    {
      "week": "2025-40",
      "n": 6
    },
    {
      "week": "2026-16",
      "n": 3
    },
    {
      "week": "2026-17",
      "n": 2
    },
    {
      "week": "2026-21",
      "n": 2
    }
  ],
  "_tags": [
    {
      "name": "v0.2.0",
      "date": "2025-08-26"
    },
    {
      "name": "v0.2.1",
      "date": "2025-09-30"
    },
    {
      "name": "v0.2.2",
      "date": "2026-04-21"
    }
  ],
  "_stars": 47,
  "_contributors": [
    {
      "user": "dfalbel",
      "count": 107,
      "uuid": 4706822
    },
    {
      "user": "t-kalinowski",
      "count": 8,
      "uuid": 8462255
    },
    {
      "user": "cregouby",
      "count": 1,
      "uuid": 10136115
    }
  ],
  "_userbio": {
    "uuid": 55406849,
    "type": "organization",
    "name": "mlverse",
    "followers": 196,
    "description": "Open source libraries to scale Data Science"
  },
  "_downloads": {
    "count": 394,
    "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/tok"
  },
  "_devurl": "https://github.com/mlverse/tok",
  "_searchresults": 19,
  "_cargo": true,
  "_topics": [
    "rust",
    "cargo"
  ],
  "_rbuild": "4.6.1",
  "_assets": [
    "extra/citation.cff",
    "extra/citation.html",
    "extra/citation.json",
    "extra/citation.txt",
    "extra/contents.json",
    "extra/NEWS.html",
    "extra/NEWS.txt",
    "extra/readme.html",
    "extra/readme.md",
    "extra/tok.html",
    "LICENSE",
    "manual.pdf"
  ],
  "_homeurl": "https://github.com/mlverse/tok",
  "_realowner": "mlverse",
  "_cranurl": true,
  "_releases": [
    {
      "version": "0.1.0",
      "date": "2023-07-06"
    },
    {
      "version": "0.1.1",
      "date": "2023-08-18"
    },
    {
      "version": "0.1.2",
      "date": "2024-06-27"
    },
    {
      "version": "0.1.3",
      "date": "2024-07-06"
    },
    {
      "version": "0.1.4",
      "date": "2024-09-04"
    },
    {
      "version": "0.2.0",
      "date": "2025-08-27"
    },
    {
      "version": "0.2.1",
      "date": "2025-10-03"
    },
    {
      "version": "0.2.2",
      "date": "2026-04-22"
    },
    {
      "version": "0.2.3",
      "date": "2026-06-21"
    }
  ],
  "_exports": [
    "decoder_byte_level",
    "encoding",
    "model_bpe",
    "model_unigram",
    "model_wordpiece",
    "normalizer_nfc",
    "normalizer_nfkc",
    "pre_tokenizer",
    "pre_tokenizer_byte_level",
    "pre_tokenizer_whitespace",
    "processor_byte_level",
    "tok_decoder",
    "tok_model",
    "tok_normalizer",
    "tok_processor",
    "tok_trainer",
    "tokenizer",
    "trainer_bpe",
    "trainer_unigram",
    "trainer_wordpiece"
  ],
  "_help": [
    {
      "page": "decoder_byte_level",
      "title": "Byte level decoder",
      "concept": [
        "decoders"
      ],
      "topics": [
        "decoder_byte_level"
      ]
    },
    {
      "page": "encoding",
      "title": "Encoding",
      "topics": [
        "encoding"
      ]
    },
    {
      "page": "model_bpe",
      "title": "BPE model",
      "concept": [
        "model"
      ],
      "topics": [
        "model_bpe"
      ]
    },
    {
      "page": "model_unigram",
      "title": "An implementation of the Unigram algorithm",
      "concept": [
        "model"
      ],
      "topics": [
        "model_unigram"
      ]
    },
    {
      "page": "model_wordpiece",
      "title": "An implementation of the WordPiece algorithm",
      "concept": [
        "model"
      ],
      "topics": [
        "model_wordpiece"
      ]
    },
    {
      "page": "normalizer_nfc",
      "title": "NFC normalizer",
      "concept": [
        "normalizers"
      ],
      "topics": [
        "normalizer_nfc"
      ]
    },
    {
      "page": "normalizer_nfkc",
      "title": "NFKC normalizer",
      "concept": [
        "normalizers"
      ],
      "topics": [
        "normalizer_nfkc"
      ]
    },
    {
      "page": "pre_tokenizer",
      "title": "Generic class for tokenizers",
      "concept": [
        "pre_tokenizer"
      ],
      "topics": [
        "pre_tokenizer"
      ]
    },
    {
      "page": "pre_tokenizer_byte_level",
      "title": "Byte level pre tokenizer",
      "concept": [
        "pre_tokenizer"
      ],
      "topics": [
        "pre_tokenizer_byte_level"
      ]
    },
    {
      "page": "pre_tokenizer_whitespace",
      "title": "This pre-tokenizer simply splits using the following regex: \\w+|[^\\w\\s]+",
      "concept": [
        "pre_tokenizer"
      ],
      "topics": [
        "pre_tokenizer_whitespace"
      ]
    },
    {
      "page": "processor_byte_level",
      "title": "Byte Level post processor",
      "concept": [
        "processors"
      ],
      "topics": [
        "processor_byte_level"
      ]
    },
    {
      "page": "tok_decoder",
      "title": "Generic class for decoders",
      "concept": [
        "decoders"
      ],
      "topics": [
        "tok_decoder"
      ]
    },
    {
      "page": "tok_model",
      "title": "Generic class for tokenization models",
      "concept": [
        "model"
      ],
      "topics": [
        "tok_model"
      ]
    },
    {
      "page": "tok_normalizer",
      "title": "Generic class for normalizers",
      "concept": [
        "normalizers"
      ],
      "topics": [
        "tok_normalizer"
      ]
    },
    {
      "page": "tok_processor",
      "title": "Generic class for processors",
      "concept": [
        "processors"
      ],
      "topics": [
        "tok_processor"
      ]
    },
    {
      "page": "tok_trainer",
      "title": "Generic training class",
      "concept": [
        "trainer"
      ],
      "topics": [
        "tok_trainer"
      ]
    },
    {
      "page": "tokenizer",
      "title": "Tokenizer",
      "topics": [
        "tokenizer"
      ]
    },
    {
      "page": "trainer_bpe",
      "title": "BPE trainer",
      "concept": [
        "trainer"
      ],
      "topics": [
        "trainer_bpe"
      ]
    },
    {
      "page": "trainer_unigram",
      "title": "Unigram tokenizer trainer",
      "concept": [
        "trainer"
      ],
      "topics": [
        "trainer_unigram"
      ]
    },
    {
      "page": "trainer_wordpiece",
      "title": "WordPiece tokenizer trainer",
      "concept": [
        "trainer"
      ],
      "topics": [
        "trainer_wordpiece"
      ]
    }
  ],
  "_readme": "https://github.com/mlverse/tok/raw/HEAD/README.md",
  "_rundeps": [
    "cli",
    "R6"
  ],
  "_score": 6.126942717944228,
  "_indexed": true,
  "_nocasepkg": "tok",
  "_universes": [
    "mlverse",
    "t-kalinowski"
  ],
  "_binaries": [
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:18:08.000Z",
      "distro": "resolute",
      "arch": "aarch64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/5497dfcedb2e2876db77e311420f6f6ba0ccff6a41925998cbdd8a9fbd155397",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:18:13.000Z",
      "distro": "resolute",
      "arch": "x86_64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/0053bfab495c1a70276b01e00806b9989742ac557d20ffc7a5cce647d7bd712d",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:18:02.000Z",
      "distro": "resolute",
      "arch": "aarch64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/6b7ee7a9715c20a1809fa106ab1b370153bbe8dd0cd5f1652db6c713daf65508",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:17:46.000Z",
      "distro": "resolute",
      "arch": "x86_64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/fac0c589590fa2382245dba7b9a8fd7f6398d48b5dcea752e3d6ace693325fb6",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:16:21.000Z",
      "arch": "aarch64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/c6a2d826a2b94ab82948bb53bb9a9c0d68103ded1834b9cf0d612983da77206f",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:18:06.000Z",
      "arch": "x86_64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/cafa7fbfecd101247dac17336a61cd8c9e226b2101b4818b855ce95996b5a4e4",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:16:33.000Z",
      "arch": "aarch64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/be7e5aa5df51882d21b472e87a2bde460f70fb0912421f26ec82457160a826df",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:17:25.000Z",
      "arch": "x86_64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/31dc27a23593c65b136c598f16804603ebaea3b713771ffa49c05cbaec1c7bbd",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:16:18.000Z",
      "arch": "x86_64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/eb89c7219f89710311ff71693974b405472d4842c797a9aa02e29380dcbbf89d",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.5.3",
      "os": "win",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:16:44.000Z",
      "arch": "x86_64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/e84f287fc41d7e292df34d90838ac0748a57bb17f1d042995e2bf3f66e7e67d2",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    },
    {
      "r": "4.6.1",
      "os": "win",
      "version": "0.2.2.9000",
      "date": "2026-07-18T05:16:22.000Z",
      "arch": "x86_64",
      "commit": "f925ad65e356dc6d295b633391aa00eae9dad8fe",
      "fileid": "https://r2.ropensci.org/d341928acef1a3525aa34ad08a4391f64a66a82b04bacec173970ceb4977c0c4",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mlverse/actions/runs/29631783448"
    }
  ]
}