import 'dotenv/config'; import { drizzle } from 'drizzle-orm/node-postgres'; import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from 'fs'; import fs from 'fs-extra'; import OpenAI from 'openai'; import { join } from 'path'; import pkg from 'pg'; import { rimraf } from 'rimraf'; import sharp from 'sharp'; import winston from 'winston'; import { BusinessListing, CommercialPropertyListing, User, UserData } from '../models/db.model.js'; import { createDefaultUser, emailToDirName, KeyValueStyle } from '../models/main.model.js'; import { SelectOptionsService } from '../select-options/select-options.service.js'; import { toDrizzleUser } from '../utils.js'; import * as schema from './schema.js'; const typesOfBusiness: Array = [ { name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' }, { name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' }, { name: 'Real Estate', value: '3', icon: 'fa-solid fa-building', textColorClass: 'text-blue-400' }, { name: 'Uncategorized', value: '4', icon: 'fa-solid fa-question', textColorClass: 'text-cyan-400' }, { name: 'Retail', value: '5', icon: 'fa-solid fa-money-bill-wave', textColorClass: 'text-pink-400' }, { name: 'Oilfield SVE and MFG.', value: '6', icon: 'fa-solid fa-oil-well', textColorClass: 'text-indigo-400' }, { name: 'Service', value: '7', icon: 'fa-solid fa-umbrella', textColorClass: 'text-teal-400' }, { name: 'Advertising', value: '8', icon: 'fa-solid fa-rectangle-ad', textColorClass: 'text-orange-400' }, { name: 'Agriculture', value: '9', icon: 'fa-solid fa-wheat-awn', textColorClass: 'text-sky-400' }, { name: 'Franchise', value: '10', icon: 'fa-solid fa-star', textColorClass: 'text-purple-400' }, { name: 'Professional', value: '11', icon: 'fa-solid fa-user-gear', textColorClass: 'text-gray-400' }, { name: 'Manufacturing', value: '12', icon: 'fa-solid fa-industry', textColorClass: 'text-red-400' }, { name: 'Food and Restaurant', value: '13', icon: 'fa-solid fa-utensils', textColorClass: 'text-amber-700' }, ]; const { Pool } = pkg; const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, // Stellen Sie sicher, dass Sie Ihren API-Key als Umgebungsvariable setzen }); const connectionString = process.env.DATABASE_URL; // const pool = new Pool({connectionString}) const client = new Pool({ connectionString }); const db = drizzle(client, { schema, logger: true }); const logger = winston.createLogger({ transports: [new winston.transports.Console()], }); //Delete Content await db.delete(schema.commercials); await db.delete(schema.businesses); await db.delete(schema.users); let filePath = `./src/assets/geo.json`; const rawData = readFileSync(filePath, 'utf8'); const geos = JSON.parse(rawData); const sso = new SelectOptionsService(); //Broker filePath = `./data/broker.json`; let data: string = readFileSync(filePath, 'utf8'); const usersData: UserData[] = JSON.parse(data); // Erwartet ein Array von Objekten const generatedUserData = []; console.log(usersData.length); let i = 0, male = 0, female = 0; const targetPathProfile = `./pictures/profile`; deleteFilesOfDir(targetPathProfile); const targetPathLogo = `./pictures/logo`; deleteFilesOfDir(targetPathLogo); const targetPathProperty = `./pictures/property`; deleteFilesOfDir(targetPathProperty); fs.ensureDirSync(`./pictures/logo`); fs.ensureDirSync(`./pictures/profile`); fs.ensureDirSync(`./pictures/property`); // type UserProfile = Omit; // type NewUser = typeof users.$inferInsert; //for (const userData of usersData) { for (let index = 0; index < usersData.length; index++) { const userData = usersData[index]; const user: User = createDefaultUser('', '', ''); //{ id: undefined, firstname: '', lastname: '', email: '' }; user.licensedIn = []; userData.licensedIn.forEach(l => { console.log(l['value'], l['name']); user.licensedIn.push({ registerNo: l['value'], state: l['name'] }); }); user.areasServed = []; user.areasServed = userData.areasServed.map(l => { return { county: l.split(',')[0].trim(), state: l.split(',')[1].trim() }; }); user.hasCompanyLogo = true; user.hasProfile = true; user.firstname = userData.firstname; user.lastname = userData.lastname; user.email = userData.email; user.phoneNumber = userData.phoneNumber; user.description = userData.description; user.companyName = userData.companyName; user.companyOverview = userData.companyOverview; user.companyWebsite = userData.companyWebsite; user.companyLocation = userData.companyLocation; const [city, state] = user.companyLocation.split('-').map(e => e.trim()); const cityGeo = geos.states.find(s => s.state_code === state).cities.find(c => c.name === city); user.latitude = cityGeo.latitude; user.longitude = cityGeo.longitude; user.offeredServices = userData.offeredServices; user.gender = userData.gender; user.customerType = 'professional'; user.customerSubType = 'broker'; user.created = new Date(); user.updated = new Date(); // const createUserProfile = (user: User): UserProfile => { // const { id, created, updated, hasCompanyLogo, hasProfile, ...userProfile } = user; // return userProfile; // }; // const userProfile = createUserProfile(user); // logger.info(`${index} - ${JSON.stringify(userProfile)}`); // const embedding = await createEmbedding(JSON.stringify(userProfile)); //sleep(200); const u = await db .insert(schema.users) .values(toDrizzleUser(user)) .returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email, firstname: schema.users.firstname, lastname: schema.users.lastname }); generatedUserData.push(u[0]); i++; logger.info(`user_${index} inserted`); if (u[0].gender === 'male') { male++; const data = readFileSync(`./pictures_base/profile/Mann_${male}.jpg`); await storeProfilePicture(data, emailToDirName(u[0].email)); } else { female++; const data = readFileSync(`./pictures_base/profile/Frau_${female}.jpg`); await storeProfilePicture(data, emailToDirName(u[0].email)); } const data = readFileSync(`./pictures_base/logo/${i}.jpg`); await storeCompanyLogo(data, emailToDirName(u[0].email)); } //Corporate Listings filePath = `./data/commercials.json`; data = readFileSync(filePath, 'utf8'); const commercialJsonData = JSON.parse(data) as CommercialPropertyListing[]; // Erwartet ein Array von Objekten for (let index = 0; index < commercialJsonData.length; index++) { const commercial = commercialJsonData[index]; const id = commercial.id; delete commercial.id; const user = getRandomItem(generatedUserData); commercial.imageOrder = getFilenames(id); commercial.imagePath = emailToDirName(user.email); const insertionDate = getRandomDateWithinLastYear(); commercial.created = insertionDate; commercial.updated = insertionDate; commercial.email = user.email; commercial.draft = false; commercial.type = sso.typesOfCommercialProperty.find(e => e.oldValue === String(commercial.type)).value; const cityGeo = geos.states.find(s => s.state_code === commercial.state).cities.find(c => c.name === commercial.city); try { commercial.latitude = cityGeo.latitude; commercial.longitude = cityGeo.longitude; } catch (e) { console.log(`----------------> ERROR ${commercial.state} - ${commercial.city}`); } // const reducedCommercial = { // city: commercial.city, // description: commercial.description, // email: commercial.email, // price: commercial.price, // state: sso.locations.find(l => l.value === commercial.state)?.name, // title: commercial.title, // name: `${user.firstname} ${user.lastname}`, // }; // const embedding = await createEmbedding(JSON.stringify(reducedCommercial)); // sleep(200); const result = await db.insert(schema.commercials).values(commercial).returning(); // logger.info(`commercial_${index} inserted`); try { fs.copySync(`./pictures_base/property/${id}`, `./pictures/property/${result[0].imagePath}/${result[0].serialId}`); } catch (err) { console.log(`----- No pictures available for ${id} ------`); } } //Business Listings filePath = `./data/businesses.json`; data = readFileSync(filePath, 'utf8'); const businessJsonData = JSON.parse(data) as BusinessListing[]; // Erwartet ein Array von Objekten for (let index = 0; index < businessJsonData.length; index++) { const business = businessJsonData[index]; delete business.id; business.type = sso.typesOfBusiness.find(e => e.oldValue === String(business.type)).value; business.created = new Date(business.created); business.updated = new Date(business.created); const user = getRandomItem(generatedUserData); business.email = user.email; business.imageName = emailToDirName(user.email); const cityGeo = geos.states.find(s => s.state_code === business.state).cities.find(c => c.name === business.city); try { business.latitude = cityGeo.latitude; business.longitude = cityGeo.longitude; } catch (e) { console.log(`----------------> ERROR ${business.state} - ${business.city}`); } // const embeddingText = JSON.stringify({ // type: typesOfBusiness.find(b => b.value === String(business.type))?.name, // title: business.title, // description: business.description, // email: business.email, // city: business.city, // state: sso.locations.find(l => l.value === business.state)?.name, // price: business.price, // realEstateIncluded: business.realEstateIncluded, // leasedLocation: business.leasedLocation, // franchiseResale: business.franchiseResale, // salesRevenue: business.salesRevenue, // cashFlow: business.cashFlow, // supportAndTraining: business.supportAndTraining, // employees: business.employees, // established: business.established, // reasonForSale: business.reasonForSale, // name: `${user.firstname} ${user.lastname}`, // }); // const embedding = await createEmbedding(embeddingText); sleep(200); await db.insert(schema.businesses).values(business); } //End await client.end(); function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } async function createEmbedding(text: string): Promise { const response = await openai.embeddings.create({ model: 'text-embedding-3-small', input: text, }); return response.data[0].embedding; } function getRandomItem(arr: T[]): T { if (arr.length === 0) { throw new Error('The array is empty.'); } const randomIndex = Math.floor(Math.random() * arr.length); return arr[randomIndex]; } function getFilenames(id: string): string[] { try { let filePath = `./pictures_base/property/${id}`; return readdirSync(filePath); } catch (e) { return []; } } function getRandomDateWithinLastYear(): Date { const currentDate = new Date(); const lastYear = new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate()); const timeDiff = currentDate.getTime() - lastYear.getTime(); const randomTimeDiff = Math.random() * timeDiff; const randomDate = new Date(lastYear.getTime() + randomTimeDiff); return randomDate; } async function storeProfilePicture(buffer: Buffer, userId: string) { let quality = 50; const output = await sharp(buffer) .resize({ width: 300 }) .avif({ quality }) // Verwende AVIF //.webp({ quality }) // Verwende Webp .toBuffer(); await sharp(output).toFile(`./pictures/profile/${userId}.avif`); } async function storeCompanyLogo(buffer: Buffer, adjustedEmail: string) { let quality = 50; const output = await sharp(buffer) .resize({ width: 300 }) .avif({ quality }) // Verwende AVIF //.webp({ quality }) // Verwende Webp .toBuffer(); await sharp(output).toFile(`./pictures/logo/${adjustedEmail}.avif`); // Ersetze Dateierweiterung // await fs.outputFile(`./pictures/logo/${userId}`, file.buffer); } function deleteFilesOfDir(directoryPath) { // Überprüfen, ob das Verzeichnis existiert if (existsSync(directoryPath)) { // Den Inhalt des Verzeichnisses synchron löschen try { readdirSync(directoryPath).forEach(file => { const filePath = join(directoryPath, file); // Wenn es sich um ein Verzeichnis handelt, rekursiv löschen if (statSync(filePath).isDirectory()) { rimraf.sync(filePath); } else { // Wenn es sich um eine Datei handelt, direkt löschen unlinkSync(filePath); } }); console.log('Der Inhalt des Verzeichnisses wurde erfolgreich gelöscht.'); } catch (err) { console.error('Fehler beim Löschen des Verzeichnisses:', err); } } else { console.log('Das Verzeichnis existiert nicht.'); } }