270 lines
10 KiB
TypeScript
270 lines
10 KiB
TypeScript
import 'dotenv/config';
|
|
import { drizzle } from 'drizzle-orm/node-postgres';
|
|
import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from 'fs';
|
|
import fs from 'fs-extra';
|
|
import OpenAI from 'openai';
|
|
import { join } from 'path';
|
|
import pkg from 'pg';
|
|
import { rimraf } from 'rimraf';
|
|
import sharp from 'sharp';
|
|
import winston from 'winston';
|
|
import { BusinessListing, CommercialPropertyListing, User, UserData } from '../models/db.model.js';
|
|
import { emailToDirName, KeyValueStyle } from '../models/main.model.js';
|
|
import * as schema from './schema.js';
|
|
const typesOfBusiness: Array<KeyValueStyle> = [
|
|
{ name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' },
|
|
{ name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' },
|
|
{ name: 'Real Estate', value: '3', icon: 'fa-solid fa-building', textColorClass: 'text-blue-400' },
|
|
{ name: 'Uncategorized', value: '4', icon: 'fa-solid fa-question', textColorClass: 'text-cyan-400' },
|
|
{ name: 'Retail', value: '5', icon: 'fa-solid fa-money-bill-wave', textColorClass: 'text-pink-400' },
|
|
{ name: 'Oilfield SVE and MFG.', value: '6', icon: 'fa-solid fa-oil-well', textColorClass: 'text-indigo-400' },
|
|
{ name: 'Service', value: '7', icon: 'fa-solid fa-umbrella', textColorClass: 'text-teal-400' },
|
|
{ name: 'Advertising', value: '8', icon: 'fa-solid fa-rectangle-ad', textColorClass: 'text-orange-400' },
|
|
{ name: 'Agriculture', value: '9', icon: 'fa-solid fa-wheat-awn', textColorClass: 'text-sky-400' },
|
|
{ name: 'Franchise', value: '10', icon: 'fa-solid fa-star', textColorClass: 'text-purple-400' },
|
|
{ name: 'Professional', value: '11', icon: 'fa-solid fa-user-gear', textColorClass: 'text-gray-400' },
|
|
{ name: 'Manufacturing', value: '12', icon: 'fa-solid fa-industry', textColorClass: 'text-red-400' },
|
|
{ name: 'Food and Restaurant', value: '13', icon: 'fa-solid fa-utensils', textColorClass: 'text-amber-700' },
|
|
];
|
|
const { Pool } = pkg;
|
|
|
|
const openai = new OpenAI({
|
|
apiKey: process.env.OPENAI_API_KEY, // Stellen Sie sicher, dass Sie Ihren API-Key als Umgebungsvariable setzen
|
|
});
|
|
|
|
const connectionString = process.env.DATABASE_URL;
|
|
// const pool = new Pool({connectionString})
|
|
const client = new Pool({ connectionString });
|
|
const db = drizzle(client, { schema, logger: true });
|
|
const logger = winston.createLogger({
|
|
transports: [new winston.transports.Console()],
|
|
});
|
|
//Delete Content
|
|
await db.delete(schema.commercials);
|
|
await db.delete(schema.businesses);
|
|
await db.delete(schema.users);
|
|
|
|
//Broker
|
|
let filePath = `./data/broker.json`;
|
|
let data: string = readFileSync(filePath, 'utf8');
|
|
const usersData: UserData[] = JSON.parse(data); // Erwartet ein Array von Objekten
|
|
const generatedUserData = [];
|
|
console.log(usersData.length);
|
|
let i = 0,
|
|
male = 0,
|
|
female = 0;
|
|
const targetPathProfile = `./pictures/profile`;
|
|
deleteFilesOfDir(targetPathProfile);
|
|
const targetPathLogo = `./pictures/logo`;
|
|
deleteFilesOfDir(targetPathLogo);
|
|
const targetPathProperty = `./pictures/property`;
|
|
deleteFilesOfDir(targetPathProperty);
|
|
fs.ensureDirSync(`./pictures/logo`);
|
|
fs.ensureDirSync(`./pictures/profile`);
|
|
fs.ensureDirSync(`./pictures/property`);
|
|
type UserProfile = Omit<User, 'created' | 'updated' | 'hasCompanyLogo' | 'hasProfile' | 'id'>;
|
|
//for (const userData of usersData) {
|
|
for (let index = 0; index < usersData.length; index++) {
|
|
const userData = usersData[index];
|
|
const user: User = { firstname: '', lastname: '', email: '' };
|
|
user.licensedIn = [];
|
|
userData.licensedIn.forEach(l => {
|
|
console.log(l['value'], l['name']);
|
|
user.licensedIn.push({ registerNo: l['value'], state: l['name'] });
|
|
});
|
|
user.areasServed = [];
|
|
user.areasServed = userData.areasServed.map(l => {
|
|
return { county: l.split(',')[0].trim(), state: l.split(',')[1].trim() };
|
|
});
|
|
user.hasCompanyLogo = true;
|
|
user.hasProfile = true;
|
|
user.firstname = userData.firstname;
|
|
user.lastname = userData.lastname;
|
|
user.email = userData.email;
|
|
user.phoneNumber = userData.phoneNumber;
|
|
user.description = userData.description;
|
|
user.companyName = userData.companyName;
|
|
user.companyOverview = userData.companyOverview;
|
|
user.companyWebsite = userData.companyWebsite;
|
|
user.companyLocation = userData.companyLocation;
|
|
user.offeredServices = userData.offeredServices;
|
|
user.gender = userData.gender;
|
|
user.customerType = 'professional';
|
|
user.customerSubType = 'broker';
|
|
user.created = new Date();
|
|
user.updated = new Date();
|
|
const createUserProfile = (user: User): UserProfile => {
|
|
const { id, created, updated, hasCompanyLogo, hasProfile, ...userProfile } = user;
|
|
return userProfile;
|
|
};
|
|
const userProfile = createUserProfile(user);
|
|
logger.info(`${index} - ${JSON.stringify(userProfile)}`);
|
|
const embedding = await createEmbedding(JSON.stringify(userProfile));
|
|
sleep(500);
|
|
const u = await db
|
|
.insert(schema.users)
|
|
.values({
|
|
...user,
|
|
embedding: embedding,
|
|
})
|
|
.returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email });
|
|
// const u = await db.insert(schema.users).values(user).returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email });
|
|
generatedUserData.push(u[0]);
|
|
i++;
|
|
|
|
if (u[0].gender === 'male') {
|
|
male++;
|
|
const data = readFileSync(`./pictures_base/profile/Mann_${male}.jpg`);
|
|
await storeProfilePicture(data, emailToDirName(u[0].email));
|
|
} else {
|
|
female++;
|
|
const data = readFileSync(`./pictures_base/profile/Frau_${female}.jpg`);
|
|
await storeProfilePicture(data, emailToDirName(u[0].email));
|
|
}
|
|
const data = readFileSync(`./pictures_base/logo/${i}.jpg`);
|
|
await storeCompanyLogo(data, emailToDirName(u[0].email));
|
|
}
|
|
|
|
//Business Listings
|
|
filePath = `./data/businesses.json`;
|
|
data = readFileSync(filePath, 'utf8');
|
|
const businessJsonData = JSON.parse(data) as BusinessListing[]; // Erwartet ein Array von Objekten
|
|
for (const business of businessJsonData) {
|
|
delete business.id;
|
|
business.created = new Date(business.created);
|
|
business.updated = new Date(business.created);
|
|
const user = getRandomItem(generatedUserData);
|
|
business.email = user.email;
|
|
business.imageName = emailToDirName(user.email);
|
|
const embeddingText = JSON.stringify({
|
|
type: typesOfBusiness.find(b => b.value === String(business.type))?.name,
|
|
title: business.title,
|
|
description: business.description,
|
|
city: business.city,
|
|
state: business.state,
|
|
price: business.price,
|
|
realEstateIncluded: business.realEstateIncluded,
|
|
leasedLocation: business.leasedLocation,
|
|
franchiseResale: business.franchiseResale,
|
|
salesRevenue: business.salesRevenue,
|
|
cashFlow: business.cashFlow,
|
|
supportAndTraining: business.supportAndTraining,
|
|
employees: business.employees,
|
|
established: business.established,
|
|
reasonForSale: business.reasonForSale,
|
|
});
|
|
const embedding = await createEmbedding(embeddingText);
|
|
sleep(300);
|
|
await db.insert(schema.businesses).values({
|
|
...business,
|
|
embedding: embedding,
|
|
});
|
|
}
|
|
//Corporate Listings
|
|
filePath = `./data/commercials.json`;
|
|
data = readFileSync(filePath, 'utf8');
|
|
const commercialJsonData = JSON.parse(data) as CommercialPropertyListing[]; // Erwartet ein Array von Objekten
|
|
for (const commercial of commercialJsonData) {
|
|
const id = commercial.id;
|
|
delete commercial.id;
|
|
const user = getRandomItem(generatedUserData);
|
|
commercial.imageOrder = getFilenames(id);
|
|
commercial.imagePath = emailToDirName(user.email);
|
|
const insertionDate = getRandomDateWithinLastYear();
|
|
commercial.created = insertionDate;
|
|
commercial.updated = insertionDate;
|
|
commercial.email = user.email;
|
|
commercial.draft = false;
|
|
const result = await db.insert(schema.commercials).values(commercial).returning();
|
|
try {
|
|
fs.copySync(`./pictures_base/property/${id}`, `./pictures/property/${result[0].imagePath}/${result[0].serialId}`);
|
|
} catch (err) {
|
|
console.log(`----- No pictures available for ${id} ------`);
|
|
}
|
|
}
|
|
|
|
//End
|
|
await client.end();
|
|
|
|
function sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
async function createEmbedding(text: string): Promise<number[]> {
|
|
const response = await openai.embeddings.create({
|
|
model: 'text-embedding-3-small',
|
|
input: text,
|
|
});
|
|
return response.data[0].embedding;
|
|
}
|
|
|
|
function getRandomItem<T>(arr: T[]): T {
|
|
if (arr.length === 0) {
|
|
throw new Error('The array is empty.');
|
|
}
|
|
|
|
const randomIndex = Math.floor(Math.random() * arr.length);
|
|
return arr[randomIndex];
|
|
}
|
|
function getFilenames(id: string): string[] {
|
|
try {
|
|
let filePath = `./pictures_base/property/${id}`;
|
|
return readdirSync(filePath);
|
|
} catch (e) {
|
|
return [];
|
|
}
|
|
}
|
|
function getRandomDateWithinLastYear(): Date {
|
|
const currentDate = new Date();
|
|
const lastYear = new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate());
|
|
|
|
const timeDiff = currentDate.getTime() - lastYear.getTime();
|
|
const randomTimeDiff = Math.random() * timeDiff;
|
|
const randomDate = new Date(lastYear.getTime() + randomTimeDiff);
|
|
|
|
return randomDate;
|
|
}
|
|
async function storeProfilePicture(buffer: Buffer, userId: string) {
|
|
let quality = 50;
|
|
const output = await sharp(buffer)
|
|
.resize({ width: 300 })
|
|
.avif({ quality }) // Verwende AVIF
|
|
//.webp({ quality }) // Verwende Webp
|
|
.toBuffer();
|
|
await sharp(output).toFile(`./pictures/profile/${userId}.avif`);
|
|
}
|
|
|
|
async function storeCompanyLogo(buffer: Buffer, adjustedEmail: string) {
|
|
let quality = 50;
|
|
const output = await sharp(buffer)
|
|
.resize({ width: 300 })
|
|
.avif({ quality }) // Verwende AVIF
|
|
//.webp({ quality }) // Verwende Webp
|
|
.toBuffer();
|
|
await sharp(output).toFile(`./pictures/logo/${adjustedEmail}.avif`); // Ersetze Dateierweiterung
|
|
// await fs.outputFile(`./pictures/logo/${userId}`, file.buffer);
|
|
}
|
|
|
|
function deleteFilesOfDir(directoryPath) {
|
|
// Überprüfen, ob das Verzeichnis existiert
|
|
if (existsSync(directoryPath)) {
|
|
// Den Inhalt des Verzeichnisses synchron löschen
|
|
try {
|
|
readdirSync(directoryPath).forEach(file => {
|
|
const filePath = join(directoryPath, file);
|
|
// Wenn es sich um ein Verzeichnis handelt, rekursiv löschen
|
|
if (statSync(filePath).isDirectory()) {
|
|
rimraf.sync(filePath);
|
|
} else {
|
|
// Wenn es sich um eine Datei handelt, direkt löschen
|
|
unlinkSync(filePath);
|
|
}
|
|
});
|
|
console.log('Der Inhalt des Verzeichnisses wurde erfolgreich gelöscht.');
|
|
} catch (err) {
|
|
console.error('Fehler beim Löschen des Verzeichnisses:', err);
|
|
}
|
|
} else {
|
|
console.log('Das Verzeichnis existiert nicht.');
|
|
}
|
|
}
|