2025-09-05 14:59:21 +08:00

543 lines
21 KiB
JavaScript

import fsp from 'node:fs/promises';
import { useNuxt, createResolver, addTemplate, loadNuxtModuleInstance, useLogger, hasNuxtModule, getNuxtModuleVersion, hasNuxtModuleCompatibility, defineNuxtModule, addImports, addPlugin, addComponent, addServerHandler, addServerPlugin, addServerImportsDir } from '@nuxt/kit';
import { defu } from 'defu';
import { installNuxtSiteConfig, updateSiteConfig } from 'nuxt-site-config-kit';
import { relative } from 'pathe';
import { readPackageJSON } from 'pkg-types';
import { existsSync } from 'node:fs';
import { onDevToolsInitialized, extendServerRpc } from '@nuxt/devtools-kit';
import { provider, env } from 'std-env';
import { isInternalRoute, mergeOnKey, asArray, parseRobotsTxt, validateRobots, normalizeGroup, normaliseRobotsRouteRule } from '../dist/runtime/util.js';
const NonHelpfulBots = [
"Nuclei",
"WikiDo",
"Riddler",
"PetalBot",
"Zoominfobot",
"Go-http-client",
"Node/simplecrawler",
"CazoodleBot",
"dotbot/1.0",
"Gigabot",
"Barkrowler",
"BLEXBot",
"magpie-crawler"
];
const DEVTOOLS_UI_ROUTE = "/__nuxt-robots";
const DEVTOOLS_UI_LOCAL_PORT = 3030;
function setupDevToolsUI(options, resolve, nuxt = useNuxt()) {
const clientPath = resolve("./client");
const isProductionBuild = existsSync(clientPath);
if (isProductionBuild) {
nuxt.hook("vite:serverCreated", async (server) => {
const sirv = await import('sirv').then((r) => r.default || r);
server.middlewares.use(
DEVTOOLS_UI_ROUTE,
sirv(clientPath, { dev: true, single: true })
);
});
} else {
nuxt.hook("vite:extendConfig", (config) => {
config.server = config.server || {};
config.server.proxy = config.server.proxy || {};
config.server.proxy[DEVTOOLS_UI_ROUTE] = {
target: `http://localhost:${DEVTOOLS_UI_LOCAL_PORT}${DEVTOOLS_UI_ROUTE}`,
changeOrigin: true,
followRedirects: true,
rewrite: (path) => path.replace(DEVTOOLS_UI_ROUTE, "")
};
});
}
onDevToolsInitialized(async () => {
extendServerRpc("nuxt-robots", {});
});
nuxt.hook("devtools:customTabs", (tabs) => {
tabs.push({
// unique identifier
name: "nuxt-robots",
// title to display in the tab
title: "Robots",
// any icon from Iconify, or a URL to an image
icon: "carbon:bot",
// iframe view
view: {
type: "iframe",
src: DEVTOOLS_UI_ROUTE
}
});
});
}
function extendTypes(module, template) {
const nuxt = useNuxt();
const { resolve } = createResolver(import.meta.url);
addTemplate({
filename: `module/${module}.d.ts`,
getContents: async () => {
const typesPath = relative(resolve(nuxt.options.rootDir, nuxt.options.buildDir, "module"), resolve("runtime/types"));
const s = await template({ typesPath });
return `// Generated by ${module}
${s}
export {}
`;
}
});
nuxt.hooks.hook("prepare:types", ({ references }) => {
references.push({ path: resolve(nuxt.options.buildDir, `module/${module}.d.ts`) });
});
nuxt.hooks.hook("nitro:config", (config) => {
config.typescript = config.typescript || {};
config.typescript.tsConfig = config.typescript.tsConfig || {};
config.typescript.tsConfig.include = config.typescript.tsConfig.include || [];
config.typescript.tsConfig.include.push(`./module/${module}.d.ts`);
});
}
const autodetectableProviders = {
azure_static: "azure",
cloudflare_pages: "cloudflare-pages",
netlify: "netlify",
stormkit: "stormkit",
vercel: "vercel",
cleavr: "cleavr",
stackblitz: "stackblitz"
};
const autodetectableStaticProviders = {
netlify: "netlify-static",
vercel: "vercel-static"
};
function detectTarget(options = {}) {
return options?.static ? autodetectableStaticProviders[provider] : autodetectableProviders[provider];
}
function resolveNitroPreset(nitroConfig) {
if (provider === "stackblitz" || provider === "codesandbox")
return provider;
let preset;
if (nitroConfig && nitroConfig?.preset)
preset = nitroConfig.preset;
if (!preset)
preset = env.NITRO_PRESET || detectTarget() || "node-server";
return preset.replace("_", "-");
}
async function getNuxtModuleOptions(module, nuxt = useNuxt()) {
const moduleMeta = (typeof module === "string" ? { name: module } : await module.getMeta?.()) || {};
const { nuxtModule } = await loadNuxtModuleInstance(module, nuxt);
let moduleEntry;
for (const m of nuxt.options.modules) {
if (Array.isArray(m) && m.length >= 2) {
const _module = m[0];
const _moduleEntryName = typeof _module === "string" ? _module : (await _module.getMeta?.())?.name || "";
if (_moduleEntryName === moduleMeta.name)
moduleEntry = m;
}
}
let inlineOptions = {};
if (moduleEntry)
inlineOptions = moduleEntry[1];
if (nuxtModule.getOptions)
return nuxtModule.getOptions(inlineOptions, nuxt);
return inlineOptions;
}
function isNuxtGenerate(nuxt = useNuxt()) {
return nuxt.options._generate || [
"static",
"github-pages"
].includes(resolveNitroPreset(nuxt.options.nitro));
}
const logger = useLogger("@nuxt/robots");
function splitPathForI18nLocales(path, autoI18n) {
const locales = autoI18n.strategy === "prefix_except_default" ? autoI18n.locales.filter((l) => l.code !== autoI18n.defaultLocale) : autoI18n.locales;
if (!path || isInternalRoute(path))
return path;
const match = path.match(new RegExp(`^/(${locales.map((l) => l.code).join("|")})(.*)`));
const locale = match?.[1];
if (locale)
return path;
return [
// always add the original route to avoid redirects
path,
...locales.map((l) => `/${l.code}${path}`)
];
}
async function resolveI18nConfig() {
let nuxtI18nConfig = {};
let resolvedAutoI18n = false;
let normalisedLocales = [];
if (hasNuxtModule("@nuxtjs/i18n")) {
const i18nVersion = await getNuxtModuleVersion("@nuxtjs/i18n");
if (!await hasNuxtModuleCompatibility("@nuxtjs/i18n", ">=8"))
logger.warn(`You are using @nuxtjs/i18n v${i18nVersion}. For the best compatibility, please upgrade to @nuxtjs/i18n v8.0.0 or higher.`);
nuxtI18nConfig = await getNuxtModuleOptions("@nuxtjs/i18n") || {};
normalisedLocales = mergeOnKey((nuxtI18nConfig.locales || []).map((locale) => typeof locale === "string" ? { code: locale } : locale), "code");
const usingI18nPages = Object.keys(nuxtI18nConfig.pages || {}).length;
const hasI18nConfigForAlternatives = nuxtI18nConfig.differentDomains || usingI18nPages || nuxtI18nConfig.strategy !== "no_prefix" && nuxtI18nConfig.locales;
if (hasI18nConfigForAlternatives) {
resolvedAutoI18n = {
differentDomains: nuxtI18nConfig.differentDomains,
defaultLocale: nuxtI18nConfig.defaultLocale,
locales: normalisedLocales,
strategy: nuxtI18nConfig.strategy
};
}
}
return resolvedAutoI18n;
}
const module = defineNuxtModule({
meta: {
name: "@nuxtjs/robots",
compatibility: {
nuxt: ">=3.6.1",
bridge: false
},
configKey: "robots"
},
defaults: {
enabled: true,
credits: true,
debug: false,
allow: [],
disallow: [],
sitemap: [],
groups: [],
blockNonSeoBots: false,
mergeWithRobotsTxtPath: true,
header: true,
metaTag: true,
cacheControl: "max-age=14400, must-revalidate",
robotsEnabledValue: "index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1",
robotsDisabledValue: "noindex, nofollow",
disallowNonIndexableRoutes: true,
robotsTxt: true
},
async setup(config, nuxt) {
const { resolve } = createResolver(import.meta.url);
const { version } = await readPackageJSON(resolve("../package.json"));
logger.level = config.debug || nuxt.options.debug ? 4 : 3;
if (config.enabled === false) {
logger.debug("The module is disabled, skipping setup.");
["defineRobotMeta", "useRobotsRule"].forEach((name) => {
addImports({ name, from: resolve(`./runtime/nuxt/composables/mock`) });
});
nuxt.options.nitro = nuxt.options.nitro || {};
nuxt.options.nitro.imports = nuxt.options.nitro.imports || {};
nuxt.options.nitro.imports.presets = nuxt.options.nitro.imports.presets || [];
nuxt.options.nitro.imports.presets.push({
from: resolve("./runtime/nitro/mock-composables"),
imports: [
"getPathRobotConfig",
"getSiteRobotConfig"
]
});
return;
}
if (nuxt.options.app.baseURL?.length > 1 && config.robotsTxt) {
logger.error(`You are not allowed to generate a robots.txt with a base URL, please set \`{ robots: { robotsTxt: false } }\` in your nuxt.config.`);
config.robotsTxt = false;
}
if (config.rules) {
logger.warn("The `rules` option is deprecated, please use the `groups` option instead.");
if (!config.groups?.length) {
const group = {};
const keyMap = {
UserAgent: "userAgent",
Disallow: "disallow",
Allow: "allow"
};
const rules = asArray(config.rules);
for (const k in rules) {
const rule = rules[k];
for (const k2 in rule) {
const key = keyMap[k2] || k2;
if (key === "Sitemap") {
config.sitemap = asArray(config.sitemap);
config.sitemap.push(rule[k2]);
} else if (keyMap[k2]) {
if (group[key]) {
group[key] = asArray(group[key]);
group[key].push(rule[k2]);
} else {
group[key] = rule[k2];
}
}
}
}
config.groups.push(group);
}
}
const resolvedAutoI18n = typeof config.autoI18n === "boolean" ? false : config.autoI18n || await resolveI18nConfig();
if (config.blockNonSeoBots) {
config.groups.push({
userAgent: NonHelpfulBots,
comment: ["Block bots that don't benefit us."],
disallow: ["/"]
});
}
await installNuxtSiteConfig();
if (config.metaTag)
addPlugin({ mode: "server", src: resolve("./runtime/nuxt/plugins/robot-meta.server") });
if (config.robotsTxt && config.mergeWithRobotsTxtPath !== false) {
let usingRobotsTxtPath = "";
let robotsTxt = false;
const publicRobotsTxtPath = resolve(nuxt.options.rootDir, nuxt.options.dir.public, "robots.txt");
const validPaths = [
// public/robots.txt - This is the default, we need to move this to avoid issues
publicRobotsTxtPath,
// assets/robots.txt
resolve(nuxt.options.rootDir, nuxt.options.dir.assets, "robots.txt"),
// public/_robots.txt
resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_robots.txt"),
// public/_robots.txt
resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_robots.txt"),
// public/_dir/robots.txt
resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_dir", "robots.txt"),
// pages/_dir/robots.txt
resolve(nuxt.options.rootDir, nuxt.options.dir.pages, "_dir", "robots.txt"),
// pages/robots.txt
resolve(nuxt.options.rootDir, nuxt.options.dir.pages, "robots.txt"),
// robots.txt
resolve(nuxt.options.rootDir, "robots.txt")
];
if (config.mergeWithRobotsTxtPath === true) {
for (const path of validPaths) {
robotsTxt = await fsp.readFile(path, { encoding: "utf-8" }).catch(() => false);
if (robotsTxt) {
usingRobotsTxtPath = path;
break;
}
}
} else {
const customPath = resolve(nuxt.options.rootDir, config.mergeWithRobotsTxtPath);
if (!await fsp.stat(customPath).catch(() => false)) {
logger.error(`You provided an invalid \`mergeWithRobotsTxtPath\`, the file does not exist: ${customPath}.`);
} else {
usingRobotsTxtPath = customPath;
robotsTxt = await fsp.readFile(customPath, { encoding: "utf-8" });
}
}
if (typeof robotsTxt === "string") {
const path = relative(nuxt.options.rootDir, usingRobotsTxtPath);
logger.debug(`A robots.txt file was found at \`./${path}\`, merging config.`);
const parsedRobotsTxt = parseRobotsTxt(robotsTxt);
const { errors } = validateRobots(parsedRobotsTxt);
if (errors.length > 0) {
logger.error(`The \`./${path}\` file contains errors:`);
for (const error of errors)
logger.log(` - ${error}`);
logger.log("");
}
const wildCardGroups = parsedRobotsTxt.groups.filter((group) => asArray(group.userAgent).includes("*"));
if (wildCardGroups.some((group) => asArray(group.disallow).includes("/"))) {
logger.warn(`The \`./${path}\` is blocking indexing for all environments.`);
logger.info("It's recommended to use the `indexable` Site Config to toggle this instead.");
}
config.groups.push(...parsedRobotsTxt.groups);
const host = parsedRobotsTxt.groups.map((g) => g.host).filter(Boolean)[0];
if (host) {
updateSiteConfig({
_context: usingRobotsTxtPath,
url: host
});
}
config.sitemap = [.../* @__PURE__ */ new Set([...asArray(config.sitemap), ...parsedRobotsTxt.sitemaps])];
if (usingRobotsTxtPath === publicRobotsTxtPath) {
await fsp.rename(usingRobotsTxtPath, resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_robots.txt"));
logger.warn("Your robots.txt file was moved to `./public/_robots.txt` to avoid conflicts.");
const extraPaths = [];
for (const path2 of validPaths) {
if (path2 !== usingRobotsTxtPath)
extraPaths.push(` - ./${relative(nuxt.options.rootDir, path2)}`);
}
logger.info(`The following paths are also valid for your robots.txt:
${extraPaths.join("\n")}
`);
}
}
}
const nitroPreset = resolveNitroPreset(nuxt.options.nitro);
let usingNuxtContent = hasNuxtModule("@nuxt/content") && config.disableNuxtContentIntegration !== false;
if (usingNuxtContent) {
if (await hasNuxtModuleCompatibility("@nuxt/content", "^3")) {
logger.warn("Nuxt Robots does not work with Nuxt Content v3 yet, the integration will be disabled. Learn more at: https://nuxtseo.com/docs/robots/guides/content");
usingNuxtContent = false;
} else if (nitroPreset.startsWith("cloudflare")) {
logger.warn("The Nuxt Robots, Nuxt Content integration does not work with CloudFlare yet, the integration will be disabled. Learn more at: https://nuxtseo.com/docs/robots/guides/content");
usingNuxtContent = false;
}
}
nuxt.hook("modules:done", async () => {
config.sitemap = asArray(config.sitemap);
config.disallow = asArray(config.disallow);
config.allow = asArray(config.allow);
config.groups = config.groups.map(normalizeGroup);
const existingGroup = config.groups.find((stack) => stack.userAgent.length === 1 && stack.userAgent[0] === "*");
if (existingGroup) {
existingGroup.disallow = [.../* @__PURE__ */ new Set([...existingGroup.disallow || [], ...config.disallow])];
if (existingGroup.disallow.length > 1) {
existingGroup.disallow = existingGroup.disallow.filter((disallow) => disallow !== "");
}
existingGroup.allow = [.../* @__PURE__ */ new Set([...existingGroup.allow || [], ...config.allow])];
} else {
config.groups.unshift({
userAgent: ["*"],
disallow: config.disallow.length > 0 ? config.disallow : [""],
allow: config.allow
});
}
await nuxt.hooks.callHook("robots:config", config);
nuxt.options.routeRules = nuxt.options.routeRules || {};
if (config.header) {
Object.entries(nuxt.options.routeRules).forEach(([route, rules]) => {
const robotRule = normaliseRobotsRouteRule(rules);
if (robotRule && !robotRule.allow && robotRule.rule) {
nuxt.options.routeRules[route] = defu({
headers: {
"X-Robots-Tag": robotRule.rule
}
}, nuxt.options.routeRules?.[route]);
}
});
}
const extraDisallows = /* @__PURE__ */ new Set();
if (config.disallowNonIndexableRoutes) {
Object.entries(nuxt.options.routeRules || {}).forEach(([route, rules]) => {
const url = route.split("/").map((segment) => segment.startsWith(":") ? "*" : segment).join("/");
const robotsRule = normaliseRobotsRouteRule(rules);
if (robotsRule && !robotsRule.allow) {
extraDisallows.add(url.replaceAll("**", "*"));
}
});
}
const firstGroup = config.groups.find((group) => group.userAgent.includes("*"));
if (firstGroup)
firstGroup.disallow = [.../* @__PURE__ */ new Set([...firstGroup.disallow || [], ...extraDisallows])];
if (resolvedAutoI18n && resolvedAutoI18n.locales && resolvedAutoI18n.strategy !== "no_prefix") {
const i18n = resolvedAutoI18n;
for (const group of config.groups.filter((g) => !g._skipI18n)) {
group.allow = asArray(group.allow || []).map((path) => splitPathForI18nLocales(path, i18n)).flat();
group.disallow = asArray(group.disallow || []).map((path) => splitPathForI18nLocales(path, i18n)).flat();
}
}
config.groups = config.groups.map(normalizeGroup);
nuxt.options.runtimeConfig["nuxt-robots"] = {
version: version || "",
usingNuxtContent,
debug: config.debug,
credits: config.credits,
groups: config.groups,
sitemap: config.sitemap,
header: config.header,
robotsEnabledValue: config.robotsEnabledValue,
robotsDisabledValue: config.robotsDisabledValue,
// @ts-expect-error untyped
cacheControl: config.cacheControl
};
nuxt.options.runtimeConfig["nuxt-simple-robots"] = nuxt.options.runtimeConfig["nuxt-robots"];
});
extendTypes("nuxt-robots", ({ typesPath }) => {
return `
declare module 'nitropack' {
interface NitroApp {
_robots: {
ctx: import('${typesPath}').HookRobotsConfigContext
nuxtContentUrls?: Set<string>
},
_robotsRuleMactcher: (url: string) => string
}
interface NitroRouteRules {
/**
* @deprecated Use \`robots: <boolean>\` instead.
*/
index?: boolean
robots?: boolean | string | {
indexable: boolean
rule: string
}
}
interface NitroRouteConfig {
/**
* @deprecated Use \`robots: <boolean>\` instead.
*/
index?: boolean
robots?: boolean | string | {
indexable: boolean
rule: string
}
}
interface NitroRuntimeHooks {
'robots:config': (ctx: import('${typesPath}').HookRobotsConfigContext) => void | Promise<void>
'robots:robots-txt': (ctx: import('${typesPath}').HookRobotsTxtContext) => void | Promise<void>
}
}
declare module 'h3' {
interface H3EventContext {
robots: {
rule: string
indexable: boolean
}
}
}
`;
});
const isFirebase = nitroPreset === "firebase";
if ((isNuxtGenerate() || isFirebase && nuxt.options._build) && config.robotsTxt) {
nuxt.options.generate = nuxt.options.generate || {};
nuxt.options.generate.routes = asArray(nuxt.options.generate.routes || []);
nuxt.options.generate.routes.push("/robots.txt");
if (isFirebase)
logger.info("Firebase does not support dynamic robots.txt files. Prerendering /robots.txt.");
}
nuxt.options.optimization.treeShake.composables.client["nuxt-robots"] = ["defineRobotMeta"];
addImports({
name: "defineRobotMeta",
from: resolve("./runtime/nuxt/composables/defineRobotMeta")
});
addImports({
name: "useRobotsRule",
from: resolve("./runtime/nuxt/composables/useRobotsRule")
});
addComponent({
name: "RobotMeta",
filePath: resolve("./runtime/nuxt/components/RobotMeta")
});
if (config.robotsTxt) {
addServerHandler({
route: "/robots.txt",
handler: resolve("./runtime/nitro/server/robots-txt")
});
}
addServerHandler({
handler: resolve("./runtime/nitro/server/middleware")
});
addServerPlugin(resolve("./runtime/nitro/plugins/initContext"));
if (usingNuxtContent) {
addServerHandler({
route: "/__robots__/nuxt-content.json",
handler: resolve("./runtime/nitro/server/__robots__/nuxt-content")
});
}
if (config.debug || nuxt.options.dev) {
addServerHandler({
route: "/__robots__/debug.json",
handler: resolve("./runtime/nitro/server/__robots__/debug")
});
addServerHandler({
route: "/__robots__/debug-path.json",
handler: resolve("./runtime/nitro/server/__robots__/debug-path")
});
}
if (nuxt.options.dev)
setupDevToolsUI(config, resolve);
addServerImportsDir(resolve("./runtime/nitro/composables"));
nuxt.options.nitro.alias = nuxt.options.nitro.alias || {};
nuxt.options.nitro.alias["#internal/nuxt-simple-robots"] = resolve("./runtime/nitro/composables");
nuxt.options.nitro.alias["#internal/nuxt-robots"] = resolve("./runtime/nitro/composables");
nuxt.options.alias["#robots"] = resolve("./runtime");
}
});
export { module as default };