From cca1374c9e2a38b00f73f5a77a1656c7fc01a501 Mon Sep 17 00:00:00 2001 From: Timo Knuth Date: Wed, 21 Jan 2026 13:57:58 +0100 Subject: [PATCH] feat: Introduce Google Indexing API and IndexNow submission scripts with a unified URL gathering utility and setup guide. --- INDEXING_GUIDE.md | 87 +++++++++++++++++++++++++++++++++++++ package.json | 3 +- scripts/submit-indexnow.ts | 14 +++--- scripts/trigger-indexing.js | 66 ---------------------------- scripts/trigger-indexing.ts | 81 ++++++++++++++++++++++++++++++++++ src/lib/indexnow.ts | 1 + 6 files changed, 179 insertions(+), 73 deletions(-) create mode 100644 INDEXING_GUIDE.md delete mode 100644 scripts/trigger-indexing.js create mode 100644 scripts/trigger-indexing.ts diff --git a/INDEXING_GUIDE.md b/INDEXING_GUIDE.md new file mode 100644 index 0000000..4a99442 --- /dev/null +++ b/INDEXING_GUIDE.md @@ -0,0 +1,87 @@ +# Indexing Setup & Usage Guide + +This guide explains how to fast-track your content indexing on **Google** and **Bing/Yandex** using the provided scripts. + +> [!IMPORTANT] +> **WAIT UNTIL LIVE:** Do not run these scripts until your new URLs are live and returning a `200 OK` status. If you submit a `404` URL, it may negatively impact your crawling budget or cause errors. + +--- + +## 1. Google Indexing API + +The Google Indexing API allows you to notify Google when pages are added or removed. It is faster than waiting for the Googlebot to crawl your sitemap. + +### Prerequisites: `service_account.json` + +To use the script `scripts/trigger-indexing.js`, you need a **Service Account Key** from Google Cloud. + +1. **Go to Google Cloud Console:** [https://console.cloud.google.com/](https://console.cloud.google.com/) +2. **Create a Project:** (e.g., "QR Master Indexing"). +3. **Enable API:** Search for "Web Search Indexing API" and enable it. +4. **Create Service Account:** + * Go to "IAM & Admin" > "Service Accounts". + * Click "Create Service Account". + * Name it (e.g., "indexer"). + * Grant it the "Owner" role (simplest for this) or a custom role with Indexing permissions. +5. **Create Key:** + * Click on the newly created service account email. + * Go to "Keys" tab -> "Add Key" -> "Create new key" -> **JSON**. + * This will download a JSON file. +6. **Save Key:** + * Rename the file to `service_account.json`. + * Place it in the **root** of your project (same folder as `package.json`). + * **NOTE:** This file is ignored by git for security (`.gitignore`), so you must copy it manually if you switch laptops. +7. **Authorize in Search Console:** + * Open the JSON file and copy the `client_email` address. + * Go to **Google Search Console** property for `qrmaster.net`. + * Go to "Settings" > "Users and permissions". + * **Add User:** Paste the service account email and give it **"Owner"** permission. (This is required for the API to work). + +### How to Run + +1. **Run the script:** + ```bash + npm run trigger:indexing + ``` + *(Or manually: `npx tsx scripts/trigger-indexing.ts`)* + +2. The script will automatically fetch ALL active URLs from the project (including tools and blog posts) and submit them to Google. You should see a "Success" message for each URL. + +--- + +## 2. IndexNow (Bing, Yandex, etc.) + +IndexNow is a protocol used by Bing and others. It's much simpler than Google's API. + +### Prerequisites: API Key + +1. **Get Key:** Go to [Bing Webmaster Tools](https://www.bing.com/webmasters) or generate one at [indexnow.org](https://www.indexnow.org/). +2. **Verify Setup:** + * The key is typically a long random string (e.g., `abc123...`). + * Ensure you have a text file named after the key (e.g., `abc123....txt`) containing the key itself inside your `public/` folder so it's accessible at `https://www.qrmaster.net/abc123....txt`. + * Alternatively, set the environment variable in your `.env` file: + ``` + INDEXNOW_KEY=your_key_here + ``` + +### How to Run + +This script (`scripts/submit-indexnow.ts`) automatically gathers all meaningful URLs from your project (tools, blog posts, main pages) and submits them. + +1. Run the script: + ```bash + npm run submit:indexnow + ``` + *(Or manually: `npx tsx scripts/submit-indexnow.ts`)* + +2. It will output which URLs were submitted. + +--- + +## Summary Checklist + +- [ ] New page is published and live. +- [ ] `service_account.json` is in the project root. +- [ ] Service Account email is added as Owner in Google Search Console. +- [ ] Run `npm run trigger:indexing` (for Google). +- [ ] Run `npm run submit:indexnow` (for Bing/Yandex). diff --git a/package.json b/package.json index a27f6ff..a2634eb 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "dev": "next dev -p 3050", "build": "prisma generate && next build", + "trigger:indexing": "tsx scripts/trigger-indexing.ts", "submit:indexnow": "tsx scripts/submit-indexnow.ts", "start": "next start", "lint": "next lint", @@ -93,4 +94,4 @@ "engines": { "node": ">=18.0.0" } -} +} \ No newline at end of file diff --git a/scripts/submit-indexnow.ts b/scripts/submit-indexnow.ts index bc52125..1c12a8e 100644 --- a/scripts/submit-indexnow.ts +++ b/scripts/submit-indexnow.ts @@ -1,21 +1,23 @@ - // Helper script to run IndexNow submission -// Run with: npx tsx scripts/submit-indexnow.ts +// Run with: npm run submit:indexnow import { getAllIndexableUrls, submitToIndexNow } from '../src/lib/indexnow'; async function main() { - console.log('Gathering URLs for IndexNow submission...'); + console.log('🚀 Starting IndexNow Submission Script...'); + + console.log(' Gathering URLs for IndexNow submission...'); const urls = getAllIndexableUrls(); - console.log(`Found ${urls.length} indexable URLs.`); + console.log(` Found ${urls.length} indexable URLs.`); // Basic validation of key presence (logic can be improved) if (!process.env.INDEXNOW_KEY) { - console.warn('⚠️ WARNING: INDEXNOW_KEY environment variable is not set. Using placeholder.'); - // In production, you'd fail here. For dev/demo, we proceed but expect failure from API. + console.warn('⚠️ WARNING: INDEXNOW_KEY environment variable is not set.'); + console.warn(' The submission might fail if the key is not hardcoded in src/lib/indexnow.ts'); } await submitToIndexNow(urls); + console.log('\n✨ IndexNow submission process completed.'); } main().catch(console.error); diff --git a/scripts/trigger-indexing.js b/scripts/trigger-indexing.js deleted file mode 100644 index f700388..0000000 --- a/scripts/trigger-indexing.js +++ /dev/null @@ -1,66 +0,0 @@ -const { google } = require('googleapis'); -const fs = require('fs'); -const path = require('path'); - -// KONFIGURATION -// ========================================== -// Pfad zu deinem Service Account Key -const KEY_FILE = path.join(__dirname, '../service_account.json'); - -// Liste der URLs, die du indexieren willst -const URLS_TO_INDEX = [ - 'https://www.qrmaster.net/tools/barcode-generator', - // Füge hier weitere URLs hinzu -]; -// ========================================== - -async function runUsingServiceAccount() { - if (!fs.existsSync(KEY_FILE)) { - console.error('❌ FEHLER: service_account.json nicht gefunden!'); - console.error(' Bitte befolge die Anleitung in INDEXING_GUIDE.md und speichere den Key im Hauptordner.'); - return; - } - - console.log(`🔑 Authentifiziere mit Key-File: ${KEY_FILE}...`); - - const auth = new google.auth.GoogleAuth({ - keyFile: KEY_FILE, - scopes: ['https://www.googleapis.com/auth/indexing'], - }); - - const client = await auth.getClient(); - - // console.log(`🔑 Authentifiziere als: ${key.client_email}...`); - - try { - // await jwtClient.authorize(); // Nicht mehr nötig mit GoogleAuth - console.log('✅ Authentifizierung erfolgreich.'); - - for (const url of URLS_TO_INDEX) { - console.log(`🚀 Sende Indexierungs-Anfrage für: ${url}`); - - const result = await google.indexing('v3').urlNotifications.publish({ - auth: client, - requestBody: { - url: url, - type: 'URL_UPDATED' - } - }); - - console.log(` Status: ${result.status} ${result.statusText}`); - console.log(` Server Antwort:`, result.data); - } - - console.log('\n✨ Fertig! Google wurde benachrichtigt.'); - console.log(' Hinweis: Es kann immer noch ein paar Stunden dauern, bis Änderungen sichtbar sind.'); - - } catch (error) { - console.error('\n❌ Es ist ein Fehler aufgetreten:'); - console.error(error.message); - if (error.response) { - console.error('Details:', error.response.data); - } - } -} - -runUsingServiceAccount(); diff --git a/scripts/trigger-indexing.ts b/scripts/trigger-indexing.ts new file mode 100644 index 0000000..382d830 --- /dev/null +++ b/scripts/trigger-indexing.ts @@ -0,0 +1,81 @@ + +import { google } from 'googleapis'; +import fs from 'fs'; +import path from 'path'; +import { getAllIndexableUrls } from '../src/lib/indexnow'; + +// ========================================== +// CONFIGURATION +// ========================================== + +// Path to your Service Account Key (JSON file) +const KEY_FILE = path.join(__dirname, '../service_account.json'); + +// Urls are now fetched dynamically from src/lib/indexnow.ts +// ========================================== + +async function runUsingServiceAccount() { + console.log('🚀 Starting Google Indexing Script (All Pages)...'); + + if (!fs.existsSync(KEY_FILE)) { + console.error('\n❌ ERROR: Service Account Key not found!'); + console.error(` Expected path: ${KEY_FILE}`); + console.error(' Please follow the instructions in INDEXING_GUIDE.md to create and save the key.'); + return; + } + + console.log(`🔑 Authenticating with key file: ${path.basename(KEY_FILE)}...`); + + const auth = new google.auth.GoogleAuth({ + keyFile: KEY_FILE, + scopes: ['https://www.googleapis.com/auth/indexing'], + }); + + try { + const client = await auth.getClient(); + console.log('✅ Authentication successful.'); + + console.log(' Gathering URLs to index...'); + const allUrls = getAllIndexableUrls(); + console.log(` Found ${allUrls.length} URLs to index.`); + + for (const url of allUrls) { + console.log(`\n📄 Processing: ${url}`); + + try { + const result = await google.indexing('v3').urlNotifications.publish({ + auth: client, + requestBody: { + url: url, + type: 'URL_UPDATED' + } + }); + + console.log(` 👉 Status: ${result.status} ${result.statusText}`); + // Optional: Log more details from result.data if needed + + } catch (innerError: any) { + console.error(` ❌ Failed to index ${url}`); + if (innerError.response) { + console.error(` Reason: ${innerError.response.status} - ${JSON.stringify(innerError.response.data)}`); + // 429 = Quota exceeded + // 403 = Permission denied (check service account owner status) + } else { + console.error(` Reason: ${innerError.message}`); + } + } + + // Optional: Add a small delay to avoid hitting rate limits too fast if you have hundreds of URLs + // await new Promise(resolve => setTimeout(resolve, 500)); + } + + console.log('\n✨ Done! All requests processed.'); + console.log(' Note: Check Google Search Console for actual indexing status over time.'); + + } catch (error: any) { + console.error('\n❌ Fatal error occurred:'); + console.error(error.message); + } +} + +runUsingServiceAccount(); diff --git a/src/lib/indexnow.ts b/src/lib/indexnow.ts index 0b772a7..319d0de 100644 --- a/src/lib/indexnow.ts +++ b/src/lib/indexnow.ts @@ -48,6 +48,7 @@ export function getAllIndexableUrls(): string[] { // Free tools const freeTools = [ + 'barcode-generator', // Added as per request 'url-qr-code', 'vcard-qr-code', 'text-qr-code', 'email-qr-code', 'sms-qr-code', 'wifi-qr-code', 'crypto-qr-code', 'event-qr-code', 'facebook-qr-code', 'instagram-qr-code', 'twitter-qr-code', 'youtube-qr-code', 'whatsapp-qr-code',