{"$schema":"https://ui.shadcn.com/schema/registry-item.json","name":"lib-code-tokens","type":"registry:lib","title":"code-tokens helpers","description":"── TypeScript, JavaScript and JSX; and HTML, which is the same tags without the code around them ────────────────","dependencies":[],"registryDependencies":[],"files":[{"path":"registry/dashral/lib/code-tokens.ts","type":"registry:file","target":"components/dashral/lib/code-tokens.ts","content":"/**\n * A small highlighter for CodeBlock: it splits code into tokens of five kinds (keywords, strings, tags, numbers and\n * comments), the calm set the kit colors; everything else stays in the text color. It knows TypeScript and JavaScript\n * with JSX, JSON, CSS, HTML and shell, enough for the code a page shows, with no dependency and nothing to load. It\n * is not a parser: for other languages, or exact grammar, pass a highlighter's output to CodeBlock as children.\n */\n\nexport type CodeTokenKind = \"keyword\" | \"string\" | \"tag\" | \"number\" | \"comment\"\nexport type CodeToken = { text: string; kind?: CodeTokenKind }\n\ntype Family = \"script\" | \"markup\" | \"json\" | \"css\" | \"shell\"\n\nconst families: Record<string, Family> = {\n  js: \"script\", jsx: \"script\", ts: \"script\", tsx: \"script\", mjs: \"script\", cjs: \"script\", javascript: \"script\", typescript: \"script\",\n  html: \"markup\", xml: \"markup\", svg: \"markup\",\n  json: \"json\", jsonc: \"json\",\n  css: \"css\",\n  bash: \"shell\", sh: \"shell\", shell: \"shell\", zsh: \"shell\",\n}\n\n/** Whether CodeBlock can color this language. */\nexport function canHighlight(language: string | undefined) {\n  return Boolean(language && families[language.toLowerCase()])\n}\n\n/**\n * The code as lines of tokens, or null for a language it doesn't know. Joined back, the tokens are the code exactly.\n */\nexport function tokenizeLines(code: string, language: string | undefined): CodeToken[][] | null {\n  const family = language ? families[language.toLowerCase()] : undefined\n  if (!family) return null\n  const tokens = family === \"script\" ? scanScript(code, false) : family === \"markup\" ? scanScript(code, true) : family === \"json\" ? scanJson(code) : family === \"css\" ? scanCss(code) : scanShell(code)\n  // Every character must come back out; if one was lost, show the code plain rather than wrong.\n  if (tokens.map((token) => token.text).join(\"\") !== code) return null\n  const lines: CodeToken[][] = [[]]\n  for (const token of tokens) {\n    token.text.split(\"\\n\").forEach((part, index) => {\n      if (index) lines.push([])\n      if (part) lines[lines.length - 1]!.push({ text: part, kind: token.kind })\n    })\n  }\n  return lines\n}\n\n/** Collects tokens, joining neighbours of the same kind. */\nfunction collector() {\n  const tokens: CodeToken[] = []\n  const add = (text: string, kind?: CodeTokenKind) => {\n    if (!text) return\n    const last = tokens[tokens.length - 1]\n    if (last && last.kind === kind) last.text += text\n    else tokens.push({ text, kind })\n  }\n  return { tokens, add }\n}\n\n/** Matches a sticky pattern at `at`, or null. */\nfunction match(pattern: RegExp, code: string, at: number) {\n  pattern.lastIndex = at\n  return pattern.exec(code)?.[0] ?? null\n}\n\nconst isWordChar = (char: string | undefined) => Boolean(char && /[\\w$]/.test(char))\n\n// ── TypeScript, JavaScript and JSX; and HTML, which is the same tags without the code around them ────────────────\n\nconst space = /\\s+/y\nconst lineComment = /\\/\\/[^\\n]*/y\nconst blockComment = /\\/\\*[\\s\\S]*?(?:\\*\\/|$)/y\nconst markupComment = /<!--[\\s\\S]*?(?:-->|$)/y\nconst doctype = /<![^>]*>?/y\nconst doubleQuoted = /\"(?:\\\\[\\s\\S]|[^\"\\\\\\n])*\"?/y\nconst singleQuoted = /'(?:\\\\[\\s\\S]|[^'\\\\\\n])*'?/y\nconst tagStart = /<\\/?[A-Za-z][\\w.:-]*/y\nconst fragment = /<\\/?>/y\nconst numeric = /(?:0[xXbBoO][\\da-fA-F_]+|\\d[\\d_]*(?:\\.\\d[\\d_]*)?(?:[eE][+-]?\\d+)?)n?/y\nconst word = /[A-Za-z_$][\\w$]*/y\nconst tagFiller = /[^\\s\"'{>]+/y\nconst textRun = /[^<{]+/y\nconst markupTextRun = /[^<]+/y\n\nconst keywords = new Set(\n  \"import export default const let var function return if else for while do switch case break continue new class extends async await try catch finally throw typeof instanceof void delete yield this super static interface enum implements declare readonly public private protected abstract keyof satisfies\".split(\" \")\n)\n\n/** Words that are keywords only where the grammar puts them: `type Props`, `from \"x\"`, `x as T`, `a of b`, `k in o`. */\nfunction keywordHere(name: string, code: string, start: number, end: number) {\n  const before = code[start - 1]\n  const after = code.slice(end, end + 64)\n  // `type Props = …`, `type Box<T> = …`, `import type { X }`, `import type X from`; never `type=\"button\"` or `type: …`.\n  if (name === \"type\") return before !== \".\" && /^\\s+(?:\\{|[A-Za-z_$][\\w$]*\\s*(?:[=<]|from\\b))/.test(after)\n  if (name === \"from\") return /^\\s*[\"'`]/.test(after)\n  if (name === \"as\" || name === \"of\" || name === \"in\") return Boolean(before && /\\s/.test(before)) && /^\\s/.test(after)\n  return false\n}\n\ntype Frame = { mode: \"code\" | \"tag\" | \"text\"; depth: number; closing?: boolean }\n\nfunction scanScript(code: string, markup: boolean): CodeToken[] {\n  const { tokens, add } = collector()\n  // Where the scanner is: in code, inside a tag's angle brackets, or in the text between tags. JSX nests them: text\n  // holds {code}, code holds <tags>.\n  const stack: Frame[] = [{ mode: markup ? \"text\" : \"code\", depth: 0 }]\n  // The last thing that wasn't whitespace, in code: it decides whether \"<\" opens a tag or compares two values.\n  let last = \"\"\n  let lastWasKeyword = false\n  const tagMayStart = () => last === \"\" || lastWasKeyword || \"([{,;:?=&|!>}\".includes(last[last.length - 1]!)\n  let at = 0\n\n  while (at < code.length) {\n    const frame = stack[stack.length - 1]!\n    const char = code[at]!\n    let found: string | null\n\n    if ((found = match(space, code, at))) {\n      add(found)\n      at += found.length\n      continue\n    }\n\n    if (frame.mode === \"text\") {\n      if (markup && (found = match(markupComment, code, at))) add(found, \"comment\")\n      else if (markup && (found = match(doctype, code, at))) add(found)\n      else if ((found = match(fragment, code, at))) {\n        add(found, \"tag\")\n        if (found === \"<>\") stack.push({ mode: \"text\", depth: 0 })\n        else if (stack.length > 1) stack.pop()\n      } else if ((found = match(tagStart, code, at))) {\n        add(found, \"tag\")\n        stack.push({ mode: \"tag\", depth: 0, closing: found[1] === \"/\" })\n      } else if (char === \"{\" && !markup) {\n        add(char)\n        found = char\n        stack.push({ mode: \"code\", depth: 0 })\n        last = \"{\"\n        lastWasKeyword = false\n      } else if ((found = match(markup ? markupTextRun : textRun, code, at))) add(found)\n      else add((found = char))\n      at += found.length\n      continue\n    }\n\n    if (frame.mode === \"tag\") {\n      if (char === '\"' || char === \"'\") add((found = match(char === '\"' ? doubleQuoted : singleQuoted, code, at) ?? char), \"string\")\n      else if (char === \"{\" && !markup) {\n        add((found = char))\n        stack.push({ mode: \"code\", depth: 0 })\n        last = \"{\"\n        lastWasKeyword = false\n      } else if (char === \">\") {\n        add((found = char))\n        const selfClosing = code[at - 1] === \"/\"\n        stack.pop()\n        // A closing tag ends its element's text; an opening one begins it.\n        if (frame.closing) {\n          if (stack.length > 1 && stack[stack.length - 1]!.mode === \"text\") stack.pop()\n        } else if (!selfClosing) stack.push({ mode: \"text\", depth: 0 })\n        last = \">\"\n        lastWasKeyword = false\n      } else add((found = match(tagFiller, code, at) ?? char))\n      at += found.length\n      continue\n    }\n\n    // Code.\n    if ((found = match(lineComment, code, at) ?? match(blockComment, code, at))) {\n      add(found, \"comment\")\n      at += found.length\n      continue\n    }\n    if (char === \"`\") {\n      at = scanTemplate(code, at, add)\n      last = '\"'\n      lastWasKeyword = false\n      continue\n    }\n    // A quote right after a letter is an apostrophe in a sentence, not the start of a string.\n    if ((char === '\"' || char === \"'\") && !isWordChar(code[at - 1])) {\n      found = match(char === '\"' ? doubleQuoted : singleQuoted, code, at) ?? char\n      add(found, \"string\")\n      at += found.length\n      last = '\"'\n      lastWasKeyword = false\n      continue\n    }\n    if (char === \"<\" && tagMayStart()) {\n      if ((found = match(fragment, code, at)) === \"<>\") {\n        add(found, \"tag\")\n        stack.push({ mode: \"text\", depth: 0 })\n        at += found.length\n        continue\n      }\n      // `<T,>` and `<T extends …>` are a generic function's type parameters, not a tag.\n      if ((found = match(tagStart, code, at)) && found[1] !== \"/\" && !/^\\s*(?:,|extends\\b)/.test(code.slice(at + found.length, at + found.length + 12))) {\n        add(found, \"tag\")\n        stack.push({ mode: \"tag\", depth: 0 })\n        at += found.length\n        continue\n      }\n    }\n    if (/\\d/.test(char) && (found = match(numeric, code, at))) {\n      add(found, \"number\")\n      at += found.length\n      last = \"0\"\n      lastWasKeyword = false\n      continue\n    }\n    if ((found = match(word, code, at))) {\n      const end = at + found.length\n      // A property (`a.default`) or a key (`{ type: … }`, `default?: number`) only looks like a keyword.\n      const isKey = ((last === \"{\" || last === \",\") && /^\\s*:/.test(code.slice(end, end + 6))) || code.startsWith(\"?:\", end)\n      const isKeyword = code[at - 1] !== \".\" && !isKey && (keywords.has(found) || keywordHere(found, code, at, end))\n      add(found, isKeyword ? \"keyword\" : undefined)\n      at = end\n      last = found\n      lastWasKeyword = isKeyword\n      continue\n    }\n    if (char === \"{\") frame.depth++\n    else if (char === \"}\") {\n      // The brace that closes a JSX expression goes back to the tag or the text it sits in.\n      if (frame.depth === 0 && stack.length > 1) stack.pop()\n      else frame.depth--\n    }\n    add(char)\n    at++\n    last = char\n    lastWasKeyword = false\n  }\n  return tokens\n}\n\n/** A template string from its opening backtick: the text is a string, each ${…} is code. Returns where it ends. */\nfunction scanTemplate(code: string, start: number, add: (text: string, kind?: CodeTokenKind) => void) {\n  let at = start + 1\n  let from = start\n  while (at < code.length) {\n    const char = code[at]\n    if (char === \"\\\\\") at += 2\n    else if (char === \"`\") {\n      at++\n      break\n    } else if (char === \"$\" && code[at + 1] === \"{\") {\n      add(code.slice(from, at), \"string\")\n      let depth = 0\n      let end = at + 2\n      for (; end < code.length; end++) {\n        if (code[end] === \"{\") depth++\n        else if (code[end] === \"}\") {\n          if (depth === 0) break\n          depth--\n        }\n      }\n      add(\"${\")\n      for (const token of scanScript(code.slice(at + 2, end), false)) add(token.text, token.kind)\n      if (end < code.length) add(\"}\")\n      at = Math.min(end + 1, code.length)\n      from = at\n    } else at++\n  }\n  at = Math.min(at, code.length)\n  add(code.slice(from, at), \"string\")\n  return at\n}\n\n// ── JSON ─────────────────────────────────────────────────────────────────────────────────────────────────────────\n\nconst jsonNumber = /-?\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?/y\nconst jsonOther = /[^\"\\d/-]+/y\n\nfunction scanJson(code: string): CodeToken[] {\n  const { tokens, add } = collector()\n  let at = 0\n  while (at < code.length) {\n    const char = code[at]!\n    let found: string | null\n    if (char === '\"') add((found = match(doubleQuoted, code, at) ?? char), \"string\")\n    else if ((found = match(lineComment, code, at) ?? match(blockComment, code, at))) add(found, \"comment\")\n    else if (!isWordChar(code[at - 1]) && (found = match(jsonNumber, code, at))) add(found, \"number\")\n    else add((found = match(jsonOther, code, at) ?? char))\n    at += found.length\n  }\n  return tokens\n}\n\n// ── CSS ──────────────────────────────────────────────────────────────────────────────────────────────────────────\n\nconst cssString = /\"(?:\\\\[\\s\\S]|[^\"\\\\\\n])*\"?|'(?:\\\\[\\s\\S]|[^'\\\\\\n])*'?/y\nconst cssAtRule = /@[\\w-]+/y\nconst cssNumber = /-?(?:\\d+\\.?\\d*|\\.\\d+)(?:%|[a-zA-Z]+)?/y\n// Names, custom properties and hex colors: taken whole, so the digits inside them stay plain (h1, --space-2, #1d4ed8).\nconst cssName = /[#.]?-{0,2}[A-Za-z_][\\w-]*|#[\\da-fA-F]+/y\n\nfunction scanCss(code: string): CodeToken[] {\n  const { tokens, add } = collector()\n  let at = 0\n  while (at < code.length) {\n    const char = code[at]!\n    let found: string | null\n    if ((found = match(blockComment, code, at))) add(found, \"comment\")\n    else if ((char === '\"' || char === \"'\") && (found = match(cssString, code, at))) add(found, \"string\")\n    else if ((found = match(cssAtRule, code, at))) add(found, \"keyword\")\n    else if ((found = match(cssName, code, at))) add(found)\n    else if ((found = match(cssNumber, code, at))) add(found, \"number\")\n    else add((found = char))\n    at += found.length\n  }\n  return tokens\n}\n\n// ── Shell ────────────────────────────────────────────────────────────────────────────────────────────────────────\n\nconst shellComment = /#[^\\n]*/y\nconst shellDouble = /\"(?:\\\\[\\s\\S]|[^\"\\\\])*\"?/y\nconst shellSingle = /'[^']*'?/y\nconst shellWord = /[^\\s\"'#;|&()]+/y\n// Only where a command starts: `for f in *.ts; do …; done`, `export NODE_ENV=production`.\nconst shellKeywords = new Set(\"if then else elif fi for while until do done case esac function export local return\".split(\" \"))\n\nfunction scanShell(code: string): CodeToken[] {\n  const { tokens, add } = collector()\n  let at = 0\n  let commandStart = true\n  while (at < code.length) {\n    const char = code[at]!\n    let found: string | null\n    if (char === \"\\n\") {\n      add((found = char))\n      commandStart = true\n    } else if ((found = match(/[^\\S\\n]+/y, code, at))) add(found)\n    else if (char === \"#\" && (at === 0 || /\\s/.test(code[at - 1]!)) && (found = match(shellComment, code, at))) add(found, \"comment\")\n    else if (char === '\"' || char === \"'\") {\n      add((found = match(char === '\"' ? shellDouble : shellSingle, code, at) ?? char), \"string\")\n      commandStart = false\n    } else if ((found = match(shellWord, code, at))) {\n      add(found, commandStart && shellKeywords.has(found) ? \"keyword\" : undefined)\n      // After `then`, `do` or `else` another command starts.\n      commandStart = commandStart && [\"then\", \"do\", \"else\"].includes(found)\n    } else {\n      add((found = char))\n      commandStart = \";|&(\".includes(char)\n    }\n    at += found.length\n  }\n  return tokens\n}\n"}]}