feat: Introduce Google Indexing API and IndexNow submission scripts with a unified URL gathering utility and setup guide.
This commit is contained in:
Timo Knuth
parent
c1471830f3
commit
cca1374c9e
|
|
@ -0,0 +1,87 @@
|
|||
# Indexing Setup & Usage Guide
|
||||
|
||||
This guide explains how to fast-track your content indexing on **Google** and **Bing/Yandex** using the provided scripts.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **WAIT UNTIL LIVE:** Do not run these scripts until your new URLs are live and returning a `200 OK` status. If you submit a `404` URL, it may negatively impact your crawling budget or cause errors.
|
||||
|
||||
---
|
||||
|
||||
## 1. Google Indexing API
|
||||
|
||||
The Google Indexing API allows you to notify Google when pages are added or removed. It is faster than waiting for the Googlebot to crawl your sitemap.
|
||||
|
||||
### Prerequisites: `service_account.json`
|
||||
|
||||
To use the script `scripts/trigger-indexing.js`, you need a **Service Account Key** from Google Cloud.
|
||||
|
||||
1. **Go to Google Cloud Console:** [https://console.cloud.google.com/](https://console.cloud.google.com/)
|
||||
2. **Create a Project:** (e.g., "QR Master Indexing").
|
||||
3. **Enable API:** Search for "Web Search Indexing API" and enable it.
|
||||
4. **Create Service Account:**
|
||||
* Go to "IAM & Admin" > "Service Accounts".
|
||||
* Click "Create Service Account".
|
||||
* Name it (e.g., "indexer").
|
||||
* Grant it the "Owner" role (simplest for this) or a custom role with Indexing permissions.
|
||||
5. **Create Key:**
|
||||
* Click on the newly created service account email.
|
||||
* Go to "Keys" tab -> "Add Key" -> "Create new key" -> **JSON**.
|
||||
* This will download a JSON file.
|
||||
6. **Save Key:**
|
||||
* Rename the file to `service_account.json`.
|
||||
* Place it in the **root** of your project (same folder as `package.json`).
|
||||
* **NOTE:** This file is ignored by git for security (`.gitignore`), so you must copy it manually if you switch laptops.
|
||||
7. **Authorize in Search Console:**
|
||||
* Open the JSON file and copy the `client_email` address.
|
||||
* Go to **Google Search Console** property for `qrmaster.net`.
|
||||
* Go to "Settings" > "Users and permissions".
|
||||
* **Add User:** Paste the service account email and give it **"Owner"** permission. (This is required for the API to work).
|
||||
|
||||
### How to Run
|
||||
|
||||
1. **Run the script:**
|
||||
```bash
|
||||
npm run trigger:indexing
|
||||
```
|
||||
*(Or manually: `npx tsx scripts/trigger-indexing.ts`)*
|
||||
|
||||
2. The script will automatically fetch ALL active URLs from the project (including tools and blog posts) and submit them to Google. You should see a "Success" message for each URL.
|
||||
|
||||
---
|
||||
|
||||
## 2. IndexNow (Bing, Yandex, etc.)
|
||||
|
||||
IndexNow is a protocol used by Bing and others. It's much simpler than Google's API.
|
||||
|
||||
### Prerequisites: API Key
|
||||
|
||||
1. **Get Key:** Go to [Bing Webmaster Tools](https://www.bing.com/webmasters) or generate one at [indexnow.org](https://www.indexnow.org/).
|
||||
2. **Verify Setup:**
|
||||
* The key is typically a long random string (e.g., `abc123...`).
|
||||
* Ensure you have a text file named after the key (e.g., `abc123....txt`) containing the key itself inside your `public/` folder so it's accessible at `https://www.qrmaster.net/abc123....txt`.
|
||||
* Alternatively, set the environment variable in your `.env` file:
|
||||
```
|
||||
INDEXNOW_KEY=your_key_here
|
||||
```
|
||||
|
||||
### How to Run
|
||||
|
||||
This script (`scripts/submit-indexnow.ts`) automatically gathers all meaningful URLs from your project (tools, blog posts, main pages) and submits them.
|
||||
|
||||
1. Run the script:
|
||||
```bash
|
||||
npm run submit:indexnow
|
||||
```
|
||||
*(Or manually: `npx tsx scripts/submit-indexnow.ts`)*
|
||||
|
||||
2. It will output which URLs were submitted.
|
||||
|
||||
---
|
||||
|
||||
## Summary Checklist
|
||||
|
||||
- [ ] New page is published and live.
|
||||
- [ ] `service_account.json` is in the project root.
|
||||
- [ ] Service Account email is added as Owner in Google Search Console.
|
||||
- [ ] Run `npm run trigger:indexing` (for Google).
|
||||
- [ ] Run `npm run submit:indexnow` (for Bing/Yandex).
|
||||
|
|
@ -6,6 +6,7 @@
|
|||
"scripts": {
|
||||
"dev": "next dev -p 3050",
|
||||
"build": "prisma generate && next build",
|
||||
"trigger:indexing": "tsx scripts/trigger-indexing.ts",
|
||||
"submit:indexnow": "tsx scripts/submit-indexnow.ts",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
|
|
@ -93,4 +94,4 @@
|
|||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +1,23 @@
|
|||
|
||||
// Helper script to run IndexNow submission
|
||||
// Run with: npx tsx scripts/submit-indexnow.ts
|
||||
// Run with: npm run submit:indexnow
|
||||
|
||||
import { getAllIndexableUrls, submitToIndexNow } from '../src/lib/indexnow';
|
||||
|
||||
async function main() {
|
||||
console.log('Gathering URLs for IndexNow submission...');
|
||||
console.log('🚀 Starting IndexNow Submission Script...');
|
||||
|
||||
console.log(' Gathering URLs for IndexNow submission...');
|
||||
const urls = getAllIndexableUrls();
|
||||
console.log(`Found ${urls.length} indexable URLs.`);
|
||||
console.log(` Found ${urls.length} indexable URLs.`);
|
||||
|
||||
// Basic validation of key presence (logic can be improved)
|
||||
if (!process.env.INDEXNOW_KEY) {
|
||||
console.warn('⚠️ WARNING: INDEXNOW_KEY environment variable is not set. Using placeholder.');
|
||||
// In production, you'd fail here. For dev/demo, we proceed but expect failure from API.
|
||||
console.warn('⚠️ WARNING: INDEXNOW_KEY environment variable is not set.');
|
||||
console.warn(' The submission might fail if the key is not hardcoded in src/lib/indexnow.ts');
|
||||
}
|
||||
|
||||
await submitToIndexNow(urls);
|
||||
console.log('\n✨ IndexNow submission process completed.');
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
const { google } = require('googleapis');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// KONFIGURATION
|
||||
// ==========================================
|
||||
// Pfad zu deinem Service Account Key
|
||||
const KEY_FILE = path.join(__dirname, '../service_account.json');
|
||||
|
||||
// Liste der URLs, die du indexieren willst
|
||||
const URLS_TO_INDEX = [
|
||||
'https://www.qrmaster.net/tools/barcode-generator',
|
||||
// Füge hier weitere URLs hinzu
|
||||
];
|
||||
// ==========================================
|
||||
|
||||
async function runUsingServiceAccount() {
|
||||
if (!fs.existsSync(KEY_FILE)) {
|
||||
console.error('❌ FEHLER: service_account.json nicht gefunden!');
|
||||
console.error(' Bitte befolge die Anleitung in INDEXING_GUIDE.md und speichere den Key im Hauptordner.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`🔑 Authentifiziere mit Key-File: ${KEY_FILE}...`);
|
||||
|
||||
const auth = new google.auth.GoogleAuth({
|
||||
keyFile: KEY_FILE,
|
||||
scopes: ['https://www.googleapis.com/auth/indexing'],
|
||||
});
|
||||
|
||||
const client = await auth.getClient();
|
||||
|
||||
// console.log(`🔑 Authentifiziere als: ${key.client_email}...`);
|
||||
|
||||
try {
|
||||
// await jwtClient.authorize(); // Nicht mehr nötig mit GoogleAuth
|
||||
console.log('✅ Authentifizierung erfolgreich.');
|
||||
|
||||
for (const url of URLS_TO_INDEX) {
|
||||
console.log(`🚀 Sende Indexierungs-Anfrage für: ${url}`);
|
||||
|
||||
const result = await google.indexing('v3').urlNotifications.publish({
|
||||
auth: client,
|
||||
requestBody: {
|
||||
url: url,
|
||||
type: 'URL_UPDATED'
|
||||
}
|
||||
});
|
||||
|
||||
console.log(` Status: ${result.status} ${result.statusText}`);
|
||||
console.log(` Server Antwort:`, result.data);
|
||||
}
|
||||
|
||||
console.log('\n✨ Fertig! Google wurde benachrichtigt.');
|
||||
console.log(' Hinweis: Es kann immer noch ein paar Stunden dauern, bis Änderungen sichtbar sind.');
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Es ist ein Fehler aufgetreten:');
|
||||
console.error(error.message);
|
||||
if (error.response) {
|
||||
console.error('Details:', error.response.data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
runUsingServiceAccount();
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
|
||||
import { google } from 'googleapis';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { getAllIndexableUrls } from '../src/lib/indexnow';
|
||||
|
||||
// ==========================================
|
||||
// CONFIGURATION
|
||||
// ==========================================
|
||||
|
||||
// Path to your Service Account Key (JSON file)
|
||||
const KEY_FILE = path.join(__dirname, '../service_account.json');
|
||||
|
||||
// Urls are now fetched dynamically from src/lib/indexnow.ts
|
||||
// ==========================================
|
||||
|
||||
async function runUsingServiceAccount() {
|
||||
console.log('🚀 Starting Google Indexing Script (All Pages)...');
|
||||
|
||||
if (!fs.existsSync(KEY_FILE)) {
|
||||
console.error('\n❌ ERROR: Service Account Key not found!');
|
||||
console.error(` Expected path: ${KEY_FILE}`);
|
||||
console.error(' Please follow the instructions in INDEXING_GUIDE.md to create and save the key.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`🔑 Authenticating with key file: ${path.basename(KEY_FILE)}...`);
|
||||
|
||||
const auth = new google.auth.GoogleAuth({
|
||||
keyFile: KEY_FILE,
|
||||
scopes: ['https://www.googleapis.com/auth/indexing'],
|
||||
});
|
||||
|
||||
try {
|
||||
const client = await auth.getClient();
|
||||
console.log('✅ Authentication successful.');
|
||||
|
||||
console.log(' Gathering URLs to index...');
|
||||
const allUrls = getAllIndexableUrls();
|
||||
console.log(` Found ${allUrls.length} URLs to index.`);
|
||||
|
||||
for (const url of allUrls) {
|
||||
console.log(`\n📄 Processing: ${url}`);
|
||||
|
||||
try {
|
||||
const result = await google.indexing('v3').urlNotifications.publish({
|
||||
auth: client,
|
||||
requestBody: {
|
||||
url: url,
|
||||
type: 'URL_UPDATED'
|
||||
}
|
||||
});
|
||||
|
||||
console.log(` 👉 Status: ${result.status} ${result.statusText}`);
|
||||
// Optional: Log more details from result.data if needed
|
||||
|
||||
} catch (innerError: any) {
|
||||
console.error(` ❌ Failed to index ${url}`);
|
||||
if (innerError.response) {
|
||||
console.error(` Reason: ${innerError.response.status} - ${JSON.stringify(innerError.response.data)}`);
|
||||
// 429 = Quota exceeded
|
||||
// 403 = Permission denied (check service account owner status)
|
||||
} else {
|
||||
console.error(` Reason: ${innerError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Optional: Add a small delay to avoid hitting rate limits too fast if you have hundreds of URLs
|
||||
// await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
console.log('\n✨ Done! All requests processed.');
|
||||
console.log(' Note: Check Google Search Console for actual indexing status over time.');
|
||||
|
||||
} catch (error: any) {
|
||||
console.error('\n❌ Fatal error occurred:');
|
||||
console.error(error.message);
|
||||
}
|
||||
}
|
||||
|
||||
runUsingServiceAccount();
|
||||
|
|
@ -48,6 +48,7 @@ export function getAllIndexableUrls(): string[] {
|
|||
|
||||
// Free tools
|
||||
const freeTools = [
|
||||
'barcode-generator', // Added as per request
|
||||
'url-qr-code', 'vcard-qr-code', 'text-qr-code', 'email-qr-code', 'sms-qr-code',
|
||||
'wifi-qr-code', 'crypto-qr-code', 'event-qr-code', 'facebook-qr-code',
|
||||
'instagram-qr-code', 'twitter-qr-code', 'youtube-qr-code', 'whatsapp-qr-code',
|
||||
|
|
|
|||
Loading…
Reference in New Issue