Skip to content

Commit a6a7800

Browse files
bluwynatemoo-re
bluwy
authored and
natemoo-re
committed
Refactor shikiji syntax highlighting code (#9083)
1 parent ac84769 commit a6a7800

10 files changed

Lines changed: 198 additions & 293 deletions

File tree

‎.changeset/fast-zebras-divide.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@astrojs/markdown-remark': minor
3+
---
4+
5+
Exports `createShikiHighlighter` for low-level syntax highlighting usage

‎.changeset/mighty-zebras-clap.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'@astrojs/markdoc': patch
3+
'astro': patch
4+
---
5+
6+
Uses new `createShikiHighlighter` API from `@astrojs/markdown-remark` to avoid code duplication

‎packages/astro/components/Code.astro‎

Lines changed: 6 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,7 @@ import type {
1010
ThemeRegistration,
1111
ThemeRegistrationRaw,
1212
} from 'shikiji';
13-
import { visit } from 'unist-util-visit';
14-
import { getCachedHighlighter, replaceCssVariables } from '../dist/core/shiki.js';
13+
import { getCachedHighlighter } from '../dist/core/shiki.js';
1514
1615
interface Props {
1716
/** The code to highlight. Required. */
@@ -94,60 +93,13 @@ if (typeof lang === 'object') {
9493
9594
const highlighter = await getCachedHighlighter({
9695
langs: [lang],
97-
themes: Object.values(experimentalThemes).length ? Object.values(experimentalThemes) : [theme],
96+
theme,
97+
experimentalThemes,
98+
wrap,
9899
});
99100
100-
const themeOptions = Object.values(experimentalThemes).length
101-
? { themes: experimentalThemes }
102-
: { theme };
103-
const html = highlighter.codeToHtml(code, {
104-
lang: typeof lang === 'string' ? lang : lang.name,
105-
...themeOptions,
106-
transforms: {
107-
pre(node) {
108-
// Swap to `code` tag if inline
109-
if (inline) {
110-
node.tagName = 'code';
111-
}
112-
113-
// Cast to string as shikiji will always pass them as strings instead of any other types
114-
const classValue = (node.properties.class as string) ?? '';
115-
const styleValue = (node.properties.style as string) ?? '';
116-
117-
// Replace "shiki" class naming with "astro-code"
118-
node.properties.class = classValue.replace(/shiki/g, 'astro-code');
119-
120-
// Handle code wrapping
121-
// if wrap=null, do nothing.
122-
if (wrap === false) {
123-
node.properties.style = styleValue + '; overflow-x: auto;';
124-
} else if (wrap === true) {
125-
node.properties.style =
126-
styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
127-
}
128-
},
129-
code(node) {
130-
if (inline) {
131-
return node.children[0] as typeof node;
132-
}
133-
},
134-
root(node) {
135-
if (Object.values(experimentalThemes).length) {
136-
return;
137-
}
138-
139-
// theme.id for shiki -> shikiji compat
140-
const themeName = typeof theme === 'string' ? theme : theme.name;
141-
if (themeName === 'css-variables') {
142-
// Replace special color tokens to CSS variables
143-
visit(node as any, 'element', (child) => {
144-
if (child.properties?.style) {
145-
child.properties.style = replaceCssVariables(child.properties.style);
146-
}
147-
});
148-
}
149-
},
150-
},
101+
const html = highlighter.highlight(code, typeof lang === 'string' ? lang : lang.name, {
102+
inline,
151103
});
152104
---
153105

‎packages/astro/src/core/errors/dev/vite.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ import * as fs from 'node:fs';
22
import { fileURLToPath } from 'node:url';
33
import { codeToHtml } from 'shikiji';
44
import type { ErrorPayload } from 'vite';
5+
import { replaceCssVariables } from '@astrojs/markdown-remark';
56
import type { ModuleLoader } from '../../module-loader/index.js';
6-
import { replaceCssVariables } from '../../shiki.js';
77
import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js';
88
import { AstroError, type ErrorWithMetadata } from '../errors.js';
99
import { createSafeError } from '../utils.js';

‎packages/astro/src/core/shiki.ts‎

Lines changed: 8 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,13 @@
1-
import { getHighlighter, type Highlighter } from 'shikiji';
1+
import {
2+
createShikiHighlighter,
3+
type ShikiHighlighter,
4+
type ShikiConfig,
5+
} from '@astrojs/markdown-remark';
26

3-
type HighlighterOptions = NonNullable<Parameters<typeof getHighlighter>[0]>;
4-
5-
const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
6-
'#000001': 'var(--astro-code-color-text)',
7-
'#000002': 'var(--astro-code-color-background)',
8-
'#000004': 'var(--astro-code-token-constant)',
9-
'#000005': 'var(--astro-code-token-string)',
10-
'#000006': 'var(--astro-code-token-comment)',
11-
'#000007': 'var(--astro-code-token-keyword)',
12-
'#000008': 'var(--astro-code-token-parameter)',
13-
'#000009': 'var(--astro-code-token-function)',
14-
'#000010': 'var(--astro-code-token-string-expression)',
15-
'#000011': 'var(--astro-code-token-punctuation)',
16-
'#000012': 'var(--astro-code-token-link)',
17-
};
18-
const COLOR_REPLACEMENT_REGEX = new RegExp(
19-
`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
20-
'g'
21-
);
22-
23-
// Caches Promise<Highlighter> for reuse when the same theme and langs are provided
7+
// Caches Promise<ShikiHighlighter> for reuse when the same theme and langs are provided
248
const cachedHighlighters = new Map();
259

26-
/**
27-
* shiki -> shikiji compat as we need to manually replace it
28-
*/
29-
export function replaceCssVariables(str: string) {
30-
return str.replace(COLOR_REPLACEMENT_REGEX, (match) => ASTRO_COLOR_REPLACEMENTS[match] || match);
31-
}
32-
33-
export function getCachedHighlighter(opts: HighlighterOptions): Promise<Highlighter> {
10+
export function getCachedHighlighter(opts: ShikiConfig): Promise<ShikiHighlighter> {
3411
// Always sort keys before stringifying to make sure objects match regardless of parameter ordering
3512
const key = JSON.stringify(opts, Object.keys(opts).sort());
3613

@@ -39,7 +16,7 @@ export function getCachedHighlighter(opts: HighlighterOptions): Promise<Highligh
3916
return cachedHighlighters.get(key);
4017
}
4118

42-
const highlighter = getHighlighter(opts);
19+
const highlighter = createShikiHighlighter(opts);
4320
cachedHighlighters.set(key, highlighter);
4421

4522
return highlighter;

‎packages/integrations/markdoc/src/extensions/shiki.ts‎

Lines changed: 5 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,107 +1,19 @@
11
import Markdoc from '@markdoc/markdoc';
2+
import { createShikiHighlighter } from '@astrojs/markdown-remark';
23
import type { ShikiConfig } from 'astro';
34
import { unescapeHTML } from 'astro/runtime/server/index.js';
4-
import { bundledLanguages, getHighlighter, type Highlighter } from 'shikiji';
55
import type { AstroMarkdocConfig } from '../config.js';
66

7-
const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
8-
'#000001': 'var(--astro-code-color-text)',
9-
'#000002': 'var(--astro-code-color-background)',
10-
'#000004': 'var(--astro-code-token-constant)',
11-
'#000005': 'var(--astro-code-token-string)',
12-
'#000006': 'var(--astro-code-token-comment)',
13-
'#000007': 'var(--astro-code-token-keyword)',
14-
'#000008': 'var(--astro-code-token-parameter)',
15-
'#000009': 'var(--astro-code-token-function)',
16-
'#000010': 'var(--astro-code-token-string-expression)',
17-
'#000011': 'var(--astro-code-token-punctuation)',
18-
'#000012': 'var(--astro-code-token-link)',
19-
};
20-
const COLOR_REPLACEMENT_REGEX = new RegExp(
21-
`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
22-
'g'
23-
);
24-
25-
const PRE_SELECTOR = /<pre class="(.*?)shiki(.*?)"/;
26-
const LINE_SELECTOR = /<span class="line"><span style="(.*?)">([\+|\-])/g;
27-
const INLINE_STYLE_SELECTOR = /style="(.*?)"/;
28-
const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g;
29-
30-
/**
31-
* Note: cache only needed for dev server reloads, internal test suites, and manual calls to `Markdoc.transform` by the user.
32-
* Otherwise, `shiki()` is only called once per build, NOT once per page, so a cache isn't needed!
33-
*/
34-
const highlighterCache = new Map<string, Highlighter>();
35-
36-
export default async function shiki({
37-
langs = [],
38-
theme = 'github-dark',
39-
wrap = false,
40-
}: ShikiConfig = {}): Promise<AstroMarkdocConfig> {
41-
const cacheId = typeof theme === 'string' ? theme : theme.name || '';
42-
let highlighter = highlighterCache.get(cacheId)!;
43-
if (!highlighter) {
44-
highlighter = await getHighlighter({
45-
langs: langs.length ? langs : Object.keys(bundledLanguages),
46-
themes: [theme],
47-
});
48-
highlighterCache.set(cacheId, highlighter);
49-
}
7+
export default async function shiki(config?: ShikiConfig): Promise<AstroMarkdocConfig> {
8+
const highlighter = await createShikiHighlighter(config);
509

5110
return {
5211
nodes: {
5312
fence: {
5413
attributes: Markdoc.nodes.fence.attributes!,
5514
transform({ attributes }) {
56-
let lang: string;
57-
58-
if (typeof attributes.language === 'string') {
59-
const langExists = highlighter
60-
.getLoadedLanguages()
61-
.includes(attributes.language as any);
62-
if (langExists) {
63-
lang = attributes.language;
64-
} else {
65-
console.warn(
66-
`[Shiki highlighter] The language "${attributes.language}" doesn't exist, falling back to plaintext.`
67-
);
68-
lang = 'plaintext';
69-
}
70-
} else {
71-
lang = 'plaintext';
72-
}
73-
74-
let html = highlighter.codeToHtml(attributes.content, { lang, theme });
75-
76-
// Q: Could these regexes match on a user's inputted code blocks?
77-
// A: Nope! All rendered HTML is properly escaped.
78-
// Ex. If a user typed `<span class="line"` into a code block,
79-
// It would become this before hitting our regexes:
80-
// &lt;span class=&quot;line&quot;
81-
82-
html = html.replace(PRE_SELECTOR, `<pre class="$1astro-code$2"`);
83-
// Add "user-select: none;" for "+"/"-" diff symbols
84-
if (attributes.language === 'diff') {
85-
html = html.replace(
86-
LINE_SELECTOR,
87-
'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
88-
);
89-
}
90-
91-
if (wrap === false) {
92-
html = html.replace(INLINE_STYLE_SELECTOR, 'style="$1; overflow-x: auto;"');
93-
} else if (wrap === true) {
94-
html = html.replace(
95-
INLINE_STYLE_SELECTOR,
96-
'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
97-
);
98-
}
99-
100-
// theme.id for shiki -> shikiji compat
101-
const themeName = typeof theme === 'string' ? theme : theme.name;
102-
if (themeName === 'css-variables') {
103-
html = html.replace(INLINE_STYLE_SELECTOR_GLOBAL, (m) => replaceCssVariables(m));
104-
}
15+
const lang = typeof attributes.language === 'string' ? attributes.language : 'plaintext';
16+
const html = highlighter.highlight(attributes.content, lang);
10517

10618
// Use `unescapeHTML` to return `HTMLString` for Astro renderer to inline as HTML
10719
return unescapeHTML(html) as any;
@@ -110,10 +22,3 @@ export default async function shiki({
11022
},
11123
};
11224
}
113-
114-
/**
115-
* shiki -> shikiji compat as we need to manually replace it
116-
*/
117-
function replaceCssVariables(str: string) {
118-
return str.replace(COLOR_REPLACEMENT_REGEX, (match) => ASTRO_COLOR_REPLACEMENTS[match] || match);
119-
}

‎packages/markdown/remark/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export { rehypeHeadingIds } from './rehype-collect-headings.js';
3232
export { remarkCollectImages } from './remark-collect-images.js';
3333
export { remarkPrism } from './remark-prism.js';
3434
export { remarkShiki } from './remark-shiki.js';
35+
export { createShikiHighlighter, replaceCssVariables, type ShikiHighlighter } from './shiki.js';
3536
export * from './types.js';
3637

3738
export const markdownConfigDefaults: Omit<Required<AstroMarkdownOptions>, 'drafts'> = {

0 commit comments

Comments
 (0)