A web crawler program for crawling Echarts official website examples implemented by Puppeter

const ECHARTS_BASE_URL = “https://echarts.apache.org/examples/en/index.html“;

function capitalizeFirstLetter(str) {
if (!str || str.length === 0) {
return “”;
}

str = str.toLowerCase();

return str.charAt(0).toUpperCase() + str.slice(1);
}

(async function () {
const browser = await puppeteer.launch();
const page = await browser.newPage();

// Navigate the page to a URL
await page.goto(ECHARTS_BASE_URL);

// Set screen size
await page.setViewport({ width: 1080, height: 1024 });

// Type into search box
// const examples = await page.type([“.example-list-panel”]);

const searchResultSelector = “.example-list-panel > div“;

const results = await page.$$(searchResultSelector);

for (const element of results) {
// gener namespace

const ele = await element.$(“.chart-type-head“);

const title = await ele.evaluate((el) => el.textContent);

let namespace = title.split(“ “).filter(Boolean);

namespace = namespace.slice(0, namespace.length – 1);

namespace = namespace
.map((item) => item.replace(“n“, “”).replace(“/“, “”))
.filter(Boolean)
.join(“”);

console.log(`${namespace} start`);

const instances = await element.$$(“.row .example-list-item“);

const components = [];

for (const instance of instances) {
// title
const titleElement = await instance.$(“.example-title“);

const subTitle = await titleElement.evaluate((el) => el.textContent);

const titles = subTitle
.split(“ “)
.filter(Boolean)
.map((item) =>
item
.replace(/+/g, “”)
.replace(/(/g, “”)
.replace(/)/g, “”)
.replace(/-/g, “”)
);

const title = titles.map((item) => capitalizeFirstLetter(item)).join(“”);

const link = await instance.$(“.example-link“);

const newPagePromise = new Promise((resolve) => {
browser.on(“targetcreated“, async (target) => {
if (target.type() === “page“) {
const targetPage = await target.page();
const url = await targetPage.url();
if (url.includes(“editor“)) {

resolve(targetPage);
}
}
});
});

await link.click();

const newPage = await newPagePromise;

await newPage.setViewport({ width: 40000, height: 20000 });

await newPage.waitForSelector(“.ace_text-layer“);

await new Promise((resolve) => {
setTimeout(() => {
resolve();
}, 3000);
});

let content = await newPage.evaluate(
() => document.querySelector(“.ace_text-layer“).innerText
);

content = content
.replace(/[]/g, “[] as any“)
.replace(/<click to see more…>/g, “”)
.replace(/var/g, “let“);

let options;

if (content.includes(“myChart“)) {
options = generOptionsWithFn({ options: content });
} else {
options = generOptions({ options: content });
}

components.push({ options, title });

await newPage.close();
}

const args = components
.filter(({ options }) => {
if (options.includes(“$“)) return false;
return true;
})
.map(({ options, title }) =>
storiesArgs({ options: options, name: title })
)
.join(“rn“);

const scripts = storiesTpl({
namespace: `Charts/${namespace}`,
components: args,
});

fs.writeFileSync(`./bots/assests/${namespace}.stories.ts`, scripts);
console.log(`${namespace} end`);
}
})();

export const storiesArgs = ({ name, options }) => ` ;
export const ${name}: Story = {
args: {
${options},
},
};`;

export const generOptionsWithFn = ({
options,
}) => `getEchartsInstance: function(myChart: any) {
let option;
${options};
return option
}`;

export const generOptions = ({ options }) => `options:(function() {
let option;
${options};
return option
})()`;