import fsp from 'node:fs/promises'; import { useNuxt, createResolver, addTemplate, loadNuxtModuleInstance, useLogger, hasNuxtModule, getNuxtModuleVersion, hasNuxtModuleCompatibility, defineNuxtModule, addImports, addPlugin, addComponent, addServerHandler, addServerPlugin, addServerImportsDir } from '@nuxt/kit'; import { defu } from 'defu'; import { installNuxtSiteConfig, updateSiteConfig } from 'nuxt-site-config-kit'; import { relative } from 'pathe'; import { readPackageJSON } from 'pkg-types'; import { existsSync } from 'node:fs'; import { onDevToolsInitialized, extendServerRpc } from '@nuxt/devtools-kit'; import { provider, env } from 'std-env'; import { isInternalRoute, mergeOnKey, asArray, parseRobotsTxt, validateRobots, normalizeGroup, normaliseRobotsRouteRule } from '../dist/runtime/util.js'; const NonHelpfulBots = [ "Nuclei", "WikiDo", "Riddler", "PetalBot", "Zoominfobot", "Go-http-client", "Node/simplecrawler", "CazoodleBot", "dotbot/1.0", "Gigabot", "Barkrowler", "BLEXBot", "magpie-crawler" ]; const DEVTOOLS_UI_ROUTE = "/__nuxt-robots"; const DEVTOOLS_UI_LOCAL_PORT = 3030; function setupDevToolsUI(options, resolve, nuxt = useNuxt()) { const clientPath = resolve("./client"); const isProductionBuild = existsSync(clientPath); if (isProductionBuild) { nuxt.hook("vite:serverCreated", async (server) => { const sirv = await import('sirv').then((r) => r.default || r); server.middlewares.use( DEVTOOLS_UI_ROUTE, sirv(clientPath, { dev: true, single: true }) ); }); } else { nuxt.hook("vite:extendConfig", (config) => { config.server = config.server || {}; config.server.proxy = config.server.proxy || {}; config.server.proxy[DEVTOOLS_UI_ROUTE] = { target: `http://localhost:${DEVTOOLS_UI_LOCAL_PORT}${DEVTOOLS_UI_ROUTE}`, changeOrigin: true, followRedirects: true, rewrite: (path) => path.replace(DEVTOOLS_UI_ROUTE, "") }; }); } onDevToolsInitialized(async () => { extendServerRpc("nuxt-robots", {}); }); nuxt.hook("devtools:customTabs", (tabs) => { tabs.push({ // unique identifier name: "nuxt-robots", // title to display in the tab title: "Robots", // any icon from Iconify, or a URL to an image icon: "carbon:bot", // iframe view view: { type: "iframe", src: DEVTOOLS_UI_ROUTE } }); }); } function extendTypes(module, template) { const nuxt = useNuxt(); const { resolve } = createResolver(import.meta.url); addTemplate({ filename: `module/${module}.d.ts`, getContents: async () => { const typesPath = relative(resolve(nuxt.options.rootDir, nuxt.options.buildDir, "module"), resolve("runtime/types")); const s = await template({ typesPath }); return `// Generated by ${module} ${s} export {} `; } }); nuxt.hooks.hook("prepare:types", ({ references }) => { references.push({ path: resolve(nuxt.options.buildDir, `module/${module}.d.ts`) }); }); nuxt.hooks.hook("nitro:config", (config) => { config.typescript = config.typescript || {}; config.typescript.tsConfig = config.typescript.tsConfig || {}; config.typescript.tsConfig.include = config.typescript.tsConfig.include || []; config.typescript.tsConfig.include.push(`./module/${module}.d.ts`); }); } const autodetectableProviders = { azure_static: "azure", cloudflare_pages: "cloudflare-pages", netlify: "netlify", stormkit: "stormkit", vercel: "vercel", cleavr: "cleavr", stackblitz: "stackblitz" }; const autodetectableStaticProviders = { netlify: "netlify-static", vercel: "vercel-static" }; function detectTarget(options = {}) { return options?.static ? autodetectableStaticProviders[provider] : autodetectableProviders[provider]; } function resolveNitroPreset(nitroConfig) { if (provider === "stackblitz" || provider === "codesandbox") return provider; let preset; if (nitroConfig && nitroConfig?.preset) preset = nitroConfig.preset; if (!preset) preset = env.NITRO_PRESET || detectTarget() || "node-server"; return preset.replace("_", "-"); } async function getNuxtModuleOptions(module, nuxt = useNuxt()) { const moduleMeta = (typeof module === "string" ? { name: module } : await module.getMeta?.()) || {}; const { nuxtModule } = await loadNuxtModuleInstance(module, nuxt); let moduleEntry; for (const m of nuxt.options.modules) { if (Array.isArray(m) && m.length >= 2) { const _module = m[0]; const _moduleEntryName = typeof _module === "string" ? _module : (await _module.getMeta?.())?.name || ""; if (_moduleEntryName === moduleMeta.name) moduleEntry = m; } } let inlineOptions = {}; if (moduleEntry) inlineOptions = moduleEntry[1]; if (nuxtModule.getOptions) return nuxtModule.getOptions(inlineOptions, nuxt); return inlineOptions; } function isNuxtGenerate(nuxt = useNuxt()) { return nuxt.options._generate || [ "static", "github-pages" ].includes(resolveNitroPreset(nuxt.options.nitro)); } const logger = useLogger("@nuxt/robots"); function splitPathForI18nLocales(path, autoI18n) { const locales = autoI18n.strategy === "prefix_except_default" ? autoI18n.locales.filter((l) => l.code !== autoI18n.defaultLocale) : autoI18n.locales; if (!path || isInternalRoute(path)) return path; const match = path.match(new RegExp(`^/(${locales.map((l) => l.code).join("|")})(.*)`)); const locale = match?.[1]; if (locale) return path; return [ // always add the original route to avoid redirects path, ...locales.map((l) => `/${l.code}${path}`) ]; } async function resolveI18nConfig() { let nuxtI18nConfig = {}; let resolvedAutoI18n = false; let normalisedLocales = []; if (hasNuxtModule("@nuxtjs/i18n")) { const i18nVersion = await getNuxtModuleVersion("@nuxtjs/i18n"); if (!await hasNuxtModuleCompatibility("@nuxtjs/i18n", ">=8")) logger.warn(`You are using @nuxtjs/i18n v${i18nVersion}. For the best compatibility, please upgrade to @nuxtjs/i18n v8.0.0 or higher.`); nuxtI18nConfig = await getNuxtModuleOptions("@nuxtjs/i18n") || {}; normalisedLocales = mergeOnKey((nuxtI18nConfig.locales || []).map((locale) => typeof locale === "string" ? { code: locale } : locale), "code"); const usingI18nPages = Object.keys(nuxtI18nConfig.pages || {}).length; const hasI18nConfigForAlternatives = nuxtI18nConfig.differentDomains || usingI18nPages || nuxtI18nConfig.strategy !== "no_prefix" && nuxtI18nConfig.locales; if (hasI18nConfigForAlternatives) { resolvedAutoI18n = { differentDomains: nuxtI18nConfig.differentDomains, defaultLocale: nuxtI18nConfig.defaultLocale, locales: normalisedLocales, strategy: nuxtI18nConfig.strategy }; } } return resolvedAutoI18n; } const module = defineNuxtModule({ meta: { name: "@nuxtjs/robots", compatibility: { nuxt: ">=3.6.1", bridge: false }, configKey: "robots" }, defaults: { enabled: true, credits: true, debug: false, allow: [], disallow: [], sitemap: [], groups: [], blockNonSeoBots: false, mergeWithRobotsTxtPath: true, header: true, metaTag: true, cacheControl: "max-age=14400, must-revalidate", robotsEnabledValue: "index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1", robotsDisabledValue: "noindex, nofollow", disallowNonIndexableRoutes: true, robotsTxt: true }, async setup(config, nuxt) { const { resolve } = createResolver(import.meta.url); const { version } = await readPackageJSON(resolve("../package.json")); logger.level = config.debug || nuxt.options.debug ? 4 : 3; if (config.enabled === false) { logger.debug("The module is disabled, skipping setup."); ["defineRobotMeta", "useRobotsRule"].forEach((name) => { addImports({ name, from: resolve(`./runtime/nuxt/composables/mock`) }); }); nuxt.options.nitro = nuxt.options.nitro || {}; nuxt.options.nitro.imports = nuxt.options.nitro.imports || {}; nuxt.options.nitro.imports.presets = nuxt.options.nitro.imports.presets || []; nuxt.options.nitro.imports.presets.push({ from: resolve("./runtime/nitro/mock-composables"), imports: [ "getPathRobotConfig", "getSiteRobotConfig" ] }); return; } if (nuxt.options.app.baseURL?.length > 1 && config.robotsTxt) { logger.error(`You are not allowed to generate a robots.txt with a base URL, please set \`{ robots: { robotsTxt: false } }\` in your nuxt.config.`); config.robotsTxt = false; } if (config.rules) { logger.warn("The `rules` option is deprecated, please use the `groups` option instead."); if (!config.groups?.length) { const group = {}; const keyMap = { UserAgent: "userAgent", Disallow: "disallow", Allow: "allow" }; const rules = asArray(config.rules); for (const k in rules) { const rule = rules[k]; for (const k2 in rule) { const key = keyMap[k2] || k2; if (key === "Sitemap") { config.sitemap = asArray(config.sitemap); config.sitemap.push(rule[k2]); } else if (keyMap[k2]) { if (group[key]) { group[key] = asArray(group[key]); group[key].push(rule[k2]); } else { group[key] = rule[k2]; } } } } config.groups.push(group); } } const resolvedAutoI18n = typeof config.autoI18n === "boolean" ? false : config.autoI18n || await resolveI18nConfig(); if (config.blockNonSeoBots) { config.groups.push({ userAgent: NonHelpfulBots, comment: ["Block bots that don't benefit us."], disallow: ["/"] }); } await installNuxtSiteConfig(); if (config.metaTag) addPlugin({ mode: "server", src: resolve("./runtime/nuxt/plugins/robot-meta.server") }); if (config.robotsTxt && config.mergeWithRobotsTxtPath !== false) { let usingRobotsTxtPath = ""; let robotsTxt = false; const publicRobotsTxtPath = resolve(nuxt.options.rootDir, nuxt.options.dir.public, "robots.txt"); const validPaths = [ // public/robots.txt - This is the default, we need to move this to avoid issues publicRobotsTxtPath, // assets/robots.txt resolve(nuxt.options.rootDir, nuxt.options.dir.assets, "robots.txt"), // public/_robots.txt resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_robots.txt"), // public/_robots.txt resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_robots.txt"), // public/_dir/robots.txt resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_dir", "robots.txt"), // pages/_dir/robots.txt resolve(nuxt.options.rootDir, nuxt.options.dir.pages, "_dir", "robots.txt"), // pages/robots.txt resolve(nuxt.options.rootDir, nuxt.options.dir.pages, "robots.txt"), // robots.txt resolve(nuxt.options.rootDir, "robots.txt") ]; if (config.mergeWithRobotsTxtPath === true) { for (const path of validPaths) { robotsTxt = await fsp.readFile(path, { encoding: "utf-8" }).catch(() => false); if (robotsTxt) { usingRobotsTxtPath = path; break; } } } else { const customPath = resolve(nuxt.options.rootDir, config.mergeWithRobotsTxtPath); if (!await fsp.stat(customPath).catch(() => false)) { logger.error(`You provided an invalid \`mergeWithRobotsTxtPath\`, the file does not exist: ${customPath}.`); } else { usingRobotsTxtPath = customPath; robotsTxt = await fsp.readFile(customPath, { encoding: "utf-8" }); } } if (typeof robotsTxt === "string") { const path = relative(nuxt.options.rootDir, usingRobotsTxtPath); logger.debug(`A robots.txt file was found at \`./${path}\`, merging config.`); const parsedRobotsTxt = parseRobotsTxt(robotsTxt); const { errors } = validateRobots(parsedRobotsTxt); if (errors.length > 0) { logger.error(`The \`./${path}\` file contains errors:`); for (const error of errors) logger.log(` - ${error}`); logger.log(""); } const wildCardGroups = parsedRobotsTxt.groups.filter((group) => asArray(group.userAgent).includes("*")); if (wildCardGroups.some((group) => asArray(group.disallow).includes("/"))) { logger.warn(`The \`./${path}\` is blocking indexing for all environments.`); logger.info("It's recommended to use the `indexable` Site Config to toggle this instead."); } config.groups.push(...parsedRobotsTxt.groups); const host = parsedRobotsTxt.groups.map((g) => g.host).filter(Boolean)[0]; if (host) { updateSiteConfig({ _context: usingRobotsTxtPath, url: host }); } config.sitemap = [.../* @__PURE__ */ new Set([...asArray(config.sitemap), ...parsedRobotsTxt.sitemaps])]; if (usingRobotsTxtPath === publicRobotsTxtPath) { await fsp.rename(usingRobotsTxtPath, resolve(nuxt.options.rootDir, nuxt.options.dir.public, "_robots.txt")); logger.warn("Your robots.txt file was moved to `./public/_robots.txt` to avoid conflicts."); const extraPaths = []; for (const path2 of validPaths) { if (path2 !== usingRobotsTxtPath) extraPaths.push(` - ./${relative(nuxt.options.rootDir, path2)}`); } logger.info(`The following paths are also valid for your robots.txt: ${extraPaths.join("\n")} `); } } } const nitroPreset = resolveNitroPreset(nuxt.options.nitro); let usingNuxtContent = hasNuxtModule("@nuxt/content") && config.disableNuxtContentIntegration !== false; if (usingNuxtContent) { if (await hasNuxtModuleCompatibility("@nuxt/content", "^3")) { logger.warn("Nuxt Robots does not work with Nuxt Content v3 yet, the integration will be disabled. Learn more at: https://nuxtseo.com/docs/robots/guides/content"); usingNuxtContent = false; } else if (nitroPreset.startsWith("cloudflare")) { logger.warn("The Nuxt Robots, Nuxt Content integration does not work with CloudFlare yet, the integration will be disabled. Learn more at: https://nuxtseo.com/docs/robots/guides/content"); usingNuxtContent = false; } } nuxt.hook("modules:done", async () => { config.sitemap = asArray(config.sitemap); config.disallow = asArray(config.disallow); config.allow = asArray(config.allow); config.groups = config.groups.map(normalizeGroup); const existingGroup = config.groups.find((stack) => stack.userAgent.length === 1 && stack.userAgent[0] === "*"); if (existingGroup) { existingGroup.disallow = [.../* @__PURE__ */ new Set([...existingGroup.disallow || [], ...config.disallow])]; if (existingGroup.disallow.length > 1) { existingGroup.disallow = existingGroup.disallow.filter((disallow) => disallow !== ""); } existingGroup.allow = [.../* @__PURE__ */ new Set([...existingGroup.allow || [], ...config.allow])]; } else { config.groups.unshift({ userAgent: ["*"], disallow: config.disallow.length > 0 ? config.disallow : [""], allow: config.allow }); } await nuxt.hooks.callHook("robots:config", config); nuxt.options.routeRules = nuxt.options.routeRules || {}; if (config.header) { Object.entries(nuxt.options.routeRules).forEach(([route, rules]) => { const robotRule = normaliseRobotsRouteRule(rules); if (robotRule && !robotRule.allow && robotRule.rule) { nuxt.options.routeRules[route] = defu({ headers: { "X-Robots-Tag": robotRule.rule } }, nuxt.options.routeRules?.[route]); } }); } const extraDisallows = /* @__PURE__ */ new Set(); if (config.disallowNonIndexableRoutes) { Object.entries(nuxt.options.routeRules || {}).forEach(([route, rules]) => { const url = route.split("/").map((segment) => segment.startsWith(":") ? "*" : segment).join("/"); const robotsRule = normaliseRobotsRouteRule(rules); if (robotsRule && !robotsRule.allow) { extraDisallows.add(url.replaceAll("**", "*")); } }); } const firstGroup = config.groups.find((group) => group.userAgent.includes("*")); if (firstGroup) firstGroup.disallow = [.../* @__PURE__ */ new Set([...firstGroup.disallow || [], ...extraDisallows])]; if (resolvedAutoI18n && resolvedAutoI18n.locales && resolvedAutoI18n.strategy !== "no_prefix") { const i18n = resolvedAutoI18n; for (const group of config.groups.filter((g) => !g._skipI18n)) { group.allow = asArray(group.allow || []).map((path) => splitPathForI18nLocales(path, i18n)).flat(); group.disallow = asArray(group.disallow || []).map((path) => splitPathForI18nLocales(path, i18n)).flat(); } } config.groups = config.groups.map(normalizeGroup); nuxt.options.runtimeConfig["nuxt-robots"] = { version: version || "", usingNuxtContent, debug: config.debug, credits: config.credits, groups: config.groups, sitemap: config.sitemap, header: config.header, robotsEnabledValue: config.robotsEnabledValue, robotsDisabledValue: config.robotsDisabledValue, // @ts-expect-error untyped cacheControl: config.cacheControl }; nuxt.options.runtimeConfig["nuxt-simple-robots"] = nuxt.options.runtimeConfig["nuxt-robots"]; }); extendTypes("nuxt-robots", ({ typesPath }) => { return ` declare module 'nitropack' { interface NitroApp { _robots: { ctx: import('${typesPath}').HookRobotsConfigContext nuxtContentUrls?: Set }, _robotsRuleMactcher: (url: string) => string } interface NitroRouteRules { /** * @deprecated Use \`robots: \` instead. */ index?: boolean robots?: boolean | string | { indexable: boolean rule: string } } interface NitroRouteConfig { /** * @deprecated Use \`robots: \` instead. */ index?: boolean robots?: boolean | string | { indexable: boolean rule: string } } interface NitroRuntimeHooks { 'robots:config': (ctx: import('${typesPath}').HookRobotsConfigContext) => void | Promise 'robots:robots-txt': (ctx: import('${typesPath}').HookRobotsTxtContext) => void | Promise } } declare module 'h3' { interface H3EventContext { robots: { rule: string indexable: boolean } } } `; }); const isFirebase = nitroPreset === "firebase"; if ((isNuxtGenerate() || isFirebase && nuxt.options._build) && config.robotsTxt) { nuxt.options.generate = nuxt.options.generate || {}; nuxt.options.generate.routes = asArray(nuxt.options.generate.routes || []); nuxt.options.generate.routes.push("/robots.txt"); if (isFirebase) logger.info("Firebase does not support dynamic robots.txt files. Prerendering /robots.txt."); } nuxt.options.optimization.treeShake.composables.client["nuxt-robots"] = ["defineRobotMeta"]; addImports({ name: "defineRobotMeta", from: resolve("./runtime/nuxt/composables/defineRobotMeta") }); addImports({ name: "useRobotsRule", from: resolve("./runtime/nuxt/composables/useRobotsRule") }); addComponent({ name: "RobotMeta", filePath: resolve("./runtime/nuxt/components/RobotMeta") }); if (config.robotsTxt) { addServerHandler({ route: "/robots.txt", handler: resolve("./runtime/nitro/server/robots-txt") }); } addServerHandler({ handler: resolve("./runtime/nitro/server/middleware") }); addServerPlugin(resolve("./runtime/nitro/plugins/initContext")); if (usingNuxtContent) { addServerHandler({ route: "/__robots__/nuxt-content.json", handler: resolve("./runtime/nitro/server/__robots__/nuxt-content") }); } if (config.debug || nuxt.options.dev) { addServerHandler({ route: "/__robots__/debug.json", handler: resolve("./runtime/nitro/server/__robots__/debug") }); addServerHandler({ route: "/__robots__/debug-path.json", handler: resolve("./runtime/nitro/server/__robots__/debug-path") }); } if (nuxt.options.dev) setupDevToolsUI(config, resolve); addServerImportsDir(resolve("./runtime/nitro/composables")); nuxt.options.nitro.alias = nuxt.options.nitro.alias || {}; nuxt.options.nitro.alias["#internal/nuxt-simple-robots"] = resolve("./runtime/nitro/composables"); nuxt.options.nitro.alias["#internal/nuxt-robots"] = resolve("./runtime/nitro/composables"); nuxt.options.alias["#robots"] = resolve("./runtime"); } }); export { module as default };