Repository navigation
Expand file tree
/
Copy pathgatsby-node.js
More file actions
304 lines (267 loc) · 11.4 KB
/
Copy pathgatsby-node.js
File metadata and controls
304 lines (267 loc) · 11.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
const fsExtra = require('fs-extra');
const {
DOC_NAV_PAGE_ID,
NOT_FOUND_PAGE_ID,
VERSION_DROPDOWN,
SITE_URL,
LLMS_SECTIONS,
} = require('./src/configs/doc-configs');
const { getDocLinkFromEdge } = require('./src/utils/gatsby-utils.js');
/* ── Build-time Markdown generation ───────────────────────────────────────
* For every asciidoc node, convert the already-generated HTML to clean
* Markdown using cheerio (DOM pre-processing) + turndown (HTML→MD).
* The result is stored as `fields.markdownBody` on each node and exposed
* in GraphQL so CopyPageDropdown can use it instead of scraping the DOM.
*/
exports.onCreateNode = ({ node, actions }) => {
if (node.internal.type !== 'Asciidoc') return;
const { createNodeField } = actions;
const TurndownService = require('turndown');
const cheerio = require('cheerio');
const html = node.html || '';
const title = node.document?.title || node.pageAttributes?.title || '';
/* Load HTML into cheerio for pre-processing */
const $ = cheerio.load(html, { decodeEntities: false });
/* Remove anchor icon links that Asciidoctor injects next to headings */
$('a.anchor').remove();
/* Remove the embedded TOC — it adds noise to Markdown */
$('#toc').remove();
/* Convert admonition tables to readable text blocks */
$('.admonitionblock').each((_, el) => {
const type = $(el).attr('class').match(/\b(note|tip|warning|caution|important)\b/i)?.[1]?.toUpperCase() || 'NOTE';
const content = $(el).find('td.content').text().trim();
$(el).replaceWith(`<blockquote><p><strong>${type}:</strong> ${content}</p></blockquote>`);
});
/* Drop tables with no rows (usually from malformed asciidoc tables).
* turndown-plugin-gfm reads `table.rows[0].parentNode` and throws on them. */
$('table').each((_, el) => {
if (!$(el).find('tr').length) $(el).remove();
});
/* Get the cleaned HTML */
const cleanedHtml = $('body').html() || '';
/* Configure turndown */
const td = new TurndownService({
headingStyle: 'atx',
bulletListMarker: '-',
codeBlockStyle: 'fenced',
fence: '```',
});
/* GFM table plugin — renders tables as proper Markdown pipe tables */
const { tables } = require('turndown-plugin-gfm');
td.use(tables);
/* A conversion failure on one page must not fail the whole build */
let markdownBody = '';
try {
markdownBody = td.turndown(cleanedHtml);
} catch (err) {
console.warn(`Markdown conversion failed for "${title}" (${node.id}): ${err.message}`);
}
createNodeField({
node,
name: 'markdownBody',
value: markdownBody,
});
};
exports.onPostBuild = async ({ graphql, reporter }) => {
fsExtra.copyFileSync(
`${__dirname}/robots.txt`,
`${__dirname}/public/robots.txt`,
);
try {
const result = await graphql(`
query {
allAsciidoc {
edges {
node {
document { title }
pageAttributes { pageid }
fields { markdownBody }
parent {
... on File {
sourceInstanceName
relativePath
}
}
}
}
}
}
`);
if (result.errors) {
reporter.warn(`Build-time generation: GraphQL errors — ${JSON.stringify(result.errors)}`);
return;
}
// pageData keyed by pageid: { title, docPath }
// docPath is the URL-path segment (e.g. '/getting-started', '/tutorials/intro')
// derived from getDocLinkFromEdge so tutorials with subdirectories resolve correctly.
const pageData = {};
let mdCount = 0;
// Pages whose markdown exceeds this limit are truncated with a continuation link.
// Keeps all pages within the agent-score page-size threshold while preserving crawlability.
const MAX_MD_CHARS = 95_000;
result.data.allAsciidoc.edges.forEach(({ node }) => {
const pageid = node.pageAttributes?.pageid;
const title = node.document?.title;
const markdownBody = node.fields?.markdownBody;
const relativePath = node.parent?.relativePath || '';
// Auto-generated per-symbol SDK reference pages (scripts/Converter/index.ts) —
// represented in llms.txt by the single curated VisualEmbedSdk entry, not individually.
const isTypedocGenerated = relativePath.startsWith('generated/typedoc/');
if (!pageid || pageid.startsWith('nav-')) return;
const docPath = getDocLinkFromEdge({ node }); // e.g. '/getting-started' or '/tutorials/category/page'
if (title) pageData[pageid] = { title, docPath, isTypedocGenerated };
// Write static .md file — serves at /docs<docPath>.md for agent crawlers
if (markdownBody) {
let body = markdownBody;
if (body.length > MAX_MD_CHARS) {
// Trim at the last complete line within the limit
const cut = body.lastIndexOf('\n', MAX_MD_CHARS);
body = body.slice(0, cut > MAX_MD_CHARS * 0.8 ? cut : MAX_MD_CHARS);
body += `\n\n---\n\n> **Content truncated.** This page exceeds the inline size limit. View the complete documentation at [${SITE_URL}${docPath}](${SITE_URL}${docPath})\n`;
}
const header = `# ${title ?? pageid}\n\n> For the complete documentation index, see [llms.txt](${SITE_URL}/llms.txt)\n\nSource: ${SITE_URL}${docPath}\n\n`;
fsExtra.outputFileSync(
`${__dirname}/public${docPath}.md`,
header + body,
);
mdCount++;
}
});
reporter.info(`[md-gen] Wrote ${mdCount} .md files`);
// Generate llms.txt — curated sections first, then any remaining pages
const coveredIds = new Set();
const lines = [
'# ThoughtSpot Developer Documentation',
'',
'> Developer documentation for ThoughtSpot Embedded — tools, APIs, and SDKs for embedding ThoughtSpot analytics into your applications.',
'',
];
for (const section of LLMS_SECTIONS) {
const sectionLines = [];
for (const pageId of section.pageIds) {
const data = pageData[pageId];
if (data) {
sectionLines.push(`- [${data.title}](${SITE_URL}${data.docPath}.md)`);
coveredIds.add(pageId);
}
}
if (sectionLines.length) {
lines.push(`## ${section.label}`);
lines.push(...sectionLines);
lines.push('');
}
}
// Add pages that exist as Asciidoc nodes but aren't in any LLMS_SECTIONS entry.
// Excludes typedoc-generated pages — those are covered by the curated VisualEmbedSdk entry.
const uncovered = Object.entries(pageData).filter(
([id, data]) => !coveredIds.has(id) && !data.isTypedocGenerated,
);
if (uncovered.length) {
lines.push('## Additional documentation');
uncovered.forEach(([, { title, docPath }]) => lines.push(`- [${title}](${SITE_URL}${docPath}.md)`));
lines.push('');
}
fsExtra.writeFileSync(
`${__dirname}/public/llms.txt`,
lines.join('\n'),
);
reporter.info(`llms.txt: ${coveredIds.size} curated + ${uncovered.length} additional = ${coveredIds.size + uncovered.length} total pages`);
} catch (err) {
reporter.warn(`Build-time generation failed: ${err.message}`);
}
};
exports.createPages = async function ({ actions, graphql }) {
const { data } = await graphql(`
query {
allAsciidoc {
edges {
node {
html
document {
title
}
pageAttributes {
pageid
}
parent {
... on File {
name
sourceInstanceName
relativePath
}
}
}
}
}
}
`);
const namePageIdMap = {};
// Collect per-category nav HTMLs keyed by category name (pageid minus 'nav-' prefix)
const navMap = {};
const NAV_PARTIAL_PREFIX = 'nav-';
data.allAsciidoc.edges.forEach((e) => {
const {
sourceInstanceName: sourceName,
relativePath: relPath,
} = e.node.parent;
const pageId = e.node.pageAttributes.pageid;
// Collect nav-* files into the navMap (not content pages)
if (pageId && pageId.startsWith(NAV_PARTIAL_PREFIX)) {
navMap[pageId.slice(NAV_PARTIAL_PREFIX.length)] = e.node.html;
return;
}
if (sourceName === 'tutorials') {
const relPathSplit = relPath.split('/');
const pageIdSplit = pageId.split('__');
let finalPageId = pageId;
if (pageIdSplit.length > 1) {
finalPageId = pageIdSplit[1];
}
let mapPageId = `tutorials/${finalPageId}`;
if (relPathSplit.length > 1) {
mapPageId = `tutorials/${relPathSplit[0]}/${finalPageId}`;
}
namePageIdMap[e.node.parent.name] = mapPageId || NOT_FOUND_PAGE_ID;
} else {
namePageIdMap[e.node.parent.name] =
e.node.pageAttributes.pageid || NOT_FOUND_PAGE_ID;
}
});
data.allAsciidoc.edges.forEach((edge) => {
const { pageid: pageId } = edge.node.pageAttributes;
// Skip nav partial files — they are sidebar data, not content pages
if (pageId && pageId.startsWith(NAV_PARTIAL_PREFIX)) return;
const docPath = getDocLinkFromEdge(edge);
actions.createPage({
path: docPath,
component: require.resolve(
'./src/components/DevDocTemplate/index.tsx',
),
context: { pageId, navId: DOC_NAV_PAGE_ID, navMap, namePageIdMap },
});
if (pageId === 'introduction') {
actions.createPage({
path: '/',
component: require.resolve(
'./src/components/DevDocTemplate/index.tsx',
),
context: { pageId, navId: DOC_NAV_PAGE_ID, navMap, namePageIdMap },
});
}
});
VERSION_DROPDOWN.forEach((version) => {
if (version.link === ' ' || !version.iframeUrl) {
return;
}
const versionPath = version.link.startsWith('/')
? version.link.substring(1)
: version.link;
actions.createPage({
path: `/${versionPath}`,
component: require.resolve(
'./src/components/DevDocTemplate/index.tsx',
),
context: { iframeUrl: version.iframeUrl },
});
});
};