import 'dotenv/config'; import { drizzle } from 'drizzle-orm/node-postgres'; import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from 'fs'; import fs from 'fs-extra'; import OpenAI from 'openai'; import { join } from 'path'; import pkg from 'pg'; import { rimraf } from 'rimraf'; import sharp from 'sharp'; import winston from 'winston'; import { BusinessListing, CommercialPropertyListing, User, UserData } from '../models/db.model.js'; import { emailToDirName, KeyValueStyle } from '../models/main.model.js'; import * as schema from './schema.js'; const typesOfBusiness: Array = [ { name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' }, { name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' }, { name: 'Real Estate', value: '3', icon: 'fa-solid fa-building', textColorClass: 'text-blue-400' }, { name: 'Uncategorized', value: '4', icon: 'fa-solid fa-question', textColorClass: 'text-cyan-400' }, { name: 'Retail', value: '5', icon: 'fa-solid fa-money-bill-wave', textColorClass: 'text-pink-400' }, { name: 'Oilfield SVE and MFG.', value: '6', icon: 'fa-solid fa-oil-well', textColorClass: 'text-indigo-400' }, { name: 'Service', value: '7', icon: 'fa-solid fa-umbrella', textColorClass: 'text-teal-400' }, { name: 'Advertising', value: '8', icon: 'fa-solid fa-rectangle-ad', textColorClass: 'text-orange-400' }, { name: 'Agriculture', value: '9', icon: 'fa-solid fa-wheat-awn', textColorClass: 'text-sky-400' }, { name: 'Franchise', value: '10', icon: 'fa-solid fa-star', textColorClass: 'text-purple-400' }, { name: 'Professional', value: '11', icon: 'fa-solid fa-user-gear', textColorClass: 'text-gray-400' }, { name: 'Manufacturing', value: '12', icon: 'fa-solid fa-industry', textColorClass: 'text-red-400' }, { name: 'Food and Restaurant', value: '13', icon: 'fa-solid fa-utensils', textColorClass: 'text-amber-700' }, ]; const { Pool } = pkg; const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, // Stellen Sie sicher, dass Sie Ihren API-Key als Umgebungsvariable setzen }); const connectionString = process.env.DATABASE_URL; // const pool = new Pool({connectionString}) const client = new Pool({ connectionString }); const db = drizzle(client, { schema, logger: true }); const logger = winston.createLogger({ transports: [new winston.transports.Console()], }); //Delete Content await db.delete(schema.commercials); await db.delete(schema.businesses); await db.delete(schema.users); //Broker let filePath = `./data/broker.json`; let data: string = readFileSync(filePath, 'utf8'); const usersData: UserData[] = JSON.parse(data); // Erwartet ein Array von Objekten const generatedUserData = []; console.log(usersData.length); let i = 0, male = 0, female = 0; const targetPathProfile = `./pictures/profile`; deleteFilesOfDir(targetPathProfile); const targetPathLogo = `./pictures/logo`; deleteFilesOfDir(targetPathLogo); const targetPathProperty = `./pictures/property`; deleteFilesOfDir(targetPathProperty); fs.ensureDirSync(`./pictures/logo`); fs.ensureDirSync(`./pictures/profile`); fs.ensureDirSync(`./pictures/property`); type UserProfile = Omit; //for (const userData of usersData) { for (let index = 0; index < usersData.length; index++) { const userData = usersData[index]; const user: User = { firstname: '', lastname: '', email: '' }; user.licensedIn = []; userData.licensedIn.forEach(l => { console.log(l['value'], l['name']); user.licensedIn.push({ registerNo: l['value'], state: l['name'] }); }); user.areasServed = []; user.areasServed = userData.areasServed.map(l => { return { county: l.split(',')[0].trim(), state: l.split(',')[1].trim() }; }); user.hasCompanyLogo = true; user.hasProfile = true; user.firstname = userData.firstname; user.lastname = userData.lastname; user.email = userData.email; user.phoneNumber = userData.phoneNumber; user.description = userData.description; user.companyName = userData.companyName; user.companyOverview = userData.companyOverview; user.companyWebsite = userData.companyWebsite; user.companyLocation = userData.companyLocation; user.offeredServices = userData.offeredServices; user.gender = userData.gender; user.customerType = 'professional'; user.customerSubType = 'broker'; user.created = new Date(); user.updated = new Date(); const createUserProfile = (user: User): UserProfile => { const { id, created, updated, hasCompanyLogo, hasProfile, ...userProfile } = user; return userProfile; }; const userProfile = createUserProfile(user); logger.info(`${index} - ${JSON.stringify(userProfile)}`); const embedding = await createEmbedding(JSON.stringify(userProfile)); sleep(500); const u = await db .insert(schema.users) .values({ ...user, embedding: embedding, }) .returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email }); // const u = await db.insert(schema.users).values(user).returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email }); generatedUserData.push(u[0]); i++; if (u[0].gender === 'male') { male++; const data = readFileSync(`./pictures_base/profile/Mann_${male}.jpg`); await storeProfilePicture(data, emailToDirName(u[0].email)); } else { female++; const data = readFileSync(`./pictures_base/profile/Frau_${female}.jpg`); await storeProfilePicture(data, emailToDirName(u[0].email)); } const data = readFileSync(`./pictures_base/logo/${i}.jpg`); await storeCompanyLogo(data, emailToDirName(u[0].email)); } //Business Listings filePath = `./data/businesses.json`; data = readFileSync(filePath, 'utf8'); const businessJsonData = JSON.parse(data) as BusinessListing[]; // Erwartet ein Array von Objekten for (const business of businessJsonData) { delete business.id; business.created = new Date(business.created); business.updated = new Date(business.created); const user = getRandomItem(generatedUserData); business.email = user.email; business.imageName = emailToDirName(user.email); const embeddingText = JSON.stringify({ type: typesOfBusiness.find(b => b.value === String(business.type))?.name, title: business.title, description: business.description, city: business.city, state: business.state, price: business.price, realEstateIncluded: business.realEstateIncluded, leasedLocation: business.leasedLocation, franchiseResale: business.franchiseResale, salesRevenue: business.salesRevenue, cashFlow: business.cashFlow, supportAndTraining: business.supportAndTraining, employees: business.employees, established: business.established, reasonForSale: business.reasonForSale, }); const embedding = await createEmbedding(embeddingText); sleep(300); await db.insert(schema.businesses).values({ ...business, embedding: embedding, }); } //Corporate Listings filePath = `./data/commercials.json`; data = readFileSync(filePath, 'utf8'); const commercialJsonData = JSON.parse(data) as CommercialPropertyListing[]; // Erwartet ein Array von Objekten for (const commercial of commercialJsonData) { const id = commercial.id; delete commercial.id; const user = getRandomItem(generatedUserData); commercial.imageOrder = getFilenames(id); commercial.imagePath = emailToDirName(user.email); const insertionDate = getRandomDateWithinLastYear(); commercial.created = insertionDate; commercial.updated = insertionDate; commercial.email = user.email; commercial.draft = false; const result = await db.insert(schema.commercials).values(commercial).returning(); try { fs.copySync(`./pictures_base/property/${id}`, `./pictures/property/${result[0].imagePath}/${result[0].serialId}`); } catch (err) { console.log(`----- No pictures available for ${id} ------`); } } //End await client.end(); function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } async function createEmbedding(text: string): Promise { const response = await openai.embeddings.create({ model: 'text-embedding-3-small', input: text, }); return response.data[0].embedding; } function getRandomItem(arr: T[]): T { if (arr.length === 0) { throw new Error('The array is empty.'); } const randomIndex = Math.floor(Math.random() * arr.length); return arr[randomIndex]; } function getFilenames(id: string): string[] { try { let filePath = `./pictures_base/property/${id}`; return readdirSync(filePath); } catch (e) { return []; } } function getRandomDateWithinLastYear(): Date { const currentDate = new Date(); const lastYear = new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate()); const timeDiff = currentDate.getTime() - lastYear.getTime(); const randomTimeDiff = Math.random() * timeDiff; const randomDate = new Date(lastYear.getTime() + randomTimeDiff); return randomDate; } async function storeProfilePicture(buffer: Buffer, userId: string) { let quality = 50; const output = await sharp(buffer) .resize({ width: 300 }) .avif({ quality }) // Verwende AVIF //.webp({ quality }) // Verwende Webp .toBuffer(); await sharp(output).toFile(`./pictures/profile/${userId}.avif`); } async function storeCompanyLogo(buffer: Buffer, adjustedEmail: string) { let quality = 50; const output = await sharp(buffer) .resize({ width: 300 }) .avif({ quality }) // Verwende AVIF //.webp({ quality }) // Verwende Webp .toBuffer(); await sharp(output).toFile(`./pictures/logo/${adjustedEmail}.avif`); // Ersetze Dateierweiterung // await fs.outputFile(`./pictures/logo/${userId}`, file.buffer); } function deleteFilesOfDir(directoryPath) { // Überprüfen, ob das Verzeichnis existiert if (existsSync(directoryPath)) { // Den Inhalt des Verzeichnisses synchron löschen try { readdirSync(directoryPath).forEach(file => { const filePath = join(directoryPath, file); // Wenn es sich um ein Verzeichnis handelt, rekursiv löschen if (statSync(filePath).isDirectory()) { rimraf.sync(filePath); } else { // Wenn es sich um eine Datei handelt, direkt löschen unlinkSync(filePath); } }); console.log('Der Inhalt des Verzeichnisses wurde erfolgreich gelöscht.'); } catch (err) { console.error('Fehler beim Löschen des Verzeichnisses:', err); } } else { console.log('Das Verzeichnis existiert nicht.'); } }