308 lines
13 KiB
TypeScript
308 lines
13 KiB
TypeScript
import 'dotenv/config';
|
|
import { drizzle } from 'drizzle-orm/node-postgres';
|
|
import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from 'fs';
|
|
import fs from 'fs-extra';
|
|
import OpenAI from 'openai';
|
|
import { join } from 'path';
|
|
import pkg from 'pg';
|
|
import { rimraf } from 'rimraf';
|
|
import sharp from 'sharp';
|
|
import winston from 'winston';
|
|
import { BusinessListing, CommercialPropertyListing, User, UserData } from '../models/db.model.js';
|
|
import { createDefaultUser, emailToDirName, KeyValueStyle } from '../models/main.model.js';
|
|
import { SelectOptionsService } from '../select-options/select-options.service.js';
|
|
import { toDrizzleUser } from '../utils.js';
|
|
import * as schema from './schema.js';
|
|
const typesOfBusiness: Array<KeyValueStyle> = [
|
|
{ name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' },
|
|
{ name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' },
|
|
{ name: 'Real Estate', value: '3', icon: 'fa-solid fa-building', textColorClass: 'text-blue-400' },
|
|
{ name: 'Uncategorized', value: '4', icon: 'fa-solid fa-question', textColorClass: 'text-cyan-400' },
|
|
{ name: 'Retail', value: '5', icon: 'fa-solid fa-money-bill-wave', textColorClass: 'text-pink-400' },
|
|
{ name: 'Oilfield SVE and MFG.', value: '6', icon: 'fa-solid fa-oil-well', textColorClass: 'text-indigo-400' },
|
|
{ name: 'Service', value: '7', icon: 'fa-solid fa-umbrella', textColorClass: 'text-teal-400' },
|
|
{ name: 'Advertising', value: '8', icon: 'fa-solid fa-rectangle-ad', textColorClass: 'text-orange-400' },
|
|
{ name: 'Agriculture', value: '9', icon: 'fa-solid fa-wheat-awn', textColorClass: 'text-sky-400' },
|
|
{ name: 'Franchise', value: '10', icon: 'fa-solid fa-star', textColorClass: 'text-purple-400' },
|
|
{ name: 'Professional', value: '11', icon: 'fa-solid fa-user-gear', textColorClass: 'text-gray-400' },
|
|
{ name: 'Manufacturing', value: '12', icon: 'fa-solid fa-industry', textColorClass: 'text-red-400' },
|
|
{ name: 'Food and Restaurant', value: '13', icon: 'fa-solid fa-utensils', textColorClass: 'text-amber-700' },
|
|
];
|
|
const { Pool } = pkg;
|
|
|
|
const openai = new OpenAI({
|
|
apiKey: process.env.OPENAI_API_KEY, // Stellen Sie sicher, dass Sie Ihren API-Key als Umgebungsvariable setzen
|
|
});
|
|
|
|
const connectionString = process.env.DATABASE_URL;
|
|
// const pool = new Pool({connectionString})
|
|
const client = new Pool({ connectionString });
|
|
const db = drizzle(client, { schema, logger: true });
|
|
const logger = winston.createLogger({
|
|
transports: [new winston.transports.Console()],
|
|
});
|
|
//Delete Content
|
|
await db.delete(schema.commercials);
|
|
await db.delete(schema.businesses);
|
|
await db.delete(schema.users);
|
|
let filePath = `./src/assets/geo.json`;
|
|
const rawData = readFileSync(filePath, 'utf8');
|
|
const geos = JSON.parse(rawData);
|
|
|
|
const sso = new SelectOptionsService();
|
|
//Broker
|
|
filePath = `./data/broker.json`;
|
|
let data: string = readFileSync(filePath, 'utf8');
|
|
const usersData: UserData[] = JSON.parse(data); // Erwartet ein Array von Objekten
|
|
const generatedUserData = [];
|
|
console.log(usersData.length);
|
|
let i = 0,
|
|
male = 0,
|
|
female = 0;
|
|
const targetPathProfile = `./pictures/profile`;
|
|
deleteFilesOfDir(targetPathProfile);
|
|
const targetPathLogo = `./pictures/logo`;
|
|
deleteFilesOfDir(targetPathLogo);
|
|
const targetPathProperty = `./pictures/property`;
|
|
deleteFilesOfDir(targetPathProperty);
|
|
fs.ensureDirSync(`./pictures/logo`);
|
|
fs.ensureDirSync(`./pictures/profile`);
|
|
fs.ensureDirSync(`./pictures/property`);
|
|
// type UserProfile = Omit<User, 'created' | 'updated' | 'hasCompanyLogo' | 'hasProfile' | 'id'>;
|
|
|
|
// type NewUser = typeof users.$inferInsert;
|
|
//for (const userData of usersData) {
|
|
for (let index = 0; index < usersData.length; index++) {
|
|
const userData = usersData[index];
|
|
const user: User = createDefaultUser('', '', ''); //{ id: undefined, firstname: '', lastname: '', email: '' };
|
|
user.licensedIn = [];
|
|
userData.licensedIn.forEach(l => {
|
|
console.log(l['value'], l['name']);
|
|
user.licensedIn.push({ registerNo: l['value'], state: l['name'] });
|
|
});
|
|
user.areasServed = [];
|
|
user.areasServed = userData.areasServed.map(l => {
|
|
return { county: l.split(',')[0].trim(), state: l.split(',')[1].trim() };
|
|
});
|
|
user.hasCompanyLogo = true;
|
|
user.hasProfile = true;
|
|
user.firstname = userData.firstname;
|
|
user.lastname = userData.lastname;
|
|
user.email = userData.email;
|
|
user.phoneNumber = userData.phoneNumber;
|
|
user.description = userData.description;
|
|
user.companyName = userData.companyName;
|
|
user.companyOverview = userData.companyOverview;
|
|
user.companyWebsite = userData.companyWebsite;
|
|
user.companyLocation = userData.companyLocation;
|
|
const [city, state] = user.companyLocation.split('-').map(e => e.trim());
|
|
const cityGeo = geos.states.find(s => s.state_code === state).cities.find(c => c.name === city);
|
|
user.latitude = cityGeo.latitude;
|
|
user.longitude = cityGeo.longitude;
|
|
user.offeredServices = userData.offeredServices;
|
|
user.gender = userData.gender;
|
|
user.customerType = 'professional';
|
|
user.customerSubType = 'broker';
|
|
user.created = new Date();
|
|
user.updated = new Date();
|
|
// const createUserProfile = (user: User): UserProfile => {
|
|
// const { id, created, updated, hasCompanyLogo, hasProfile, ...userProfile } = user;
|
|
// return userProfile;
|
|
// };
|
|
// const userProfile = createUserProfile(user);
|
|
// logger.info(`${index} - ${JSON.stringify(userProfile)}`);
|
|
// const embedding = await createEmbedding(JSON.stringify(userProfile));
|
|
//sleep(200);
|
|
const u = await db
|
|
.insert(schema.users)
|
|
.values(toDrizzleUser(user))
|
|
.returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email, firstname: schema.users.firstname, lastname: schema.users.lastname });
|
|
generatedUserData.push(u[0]);
|
|
i++;
|
|
logger.info(`user_${index} inserted`);
|
|
if (u[0].gender === 'male') {
|
|
male++;
|
|
const data = readFileSync(`./pictures_base/profile/Mann_${male}.jpg`);
|
|
await storeProfilePicture(data, emailToDirName(u[0].email));
|
|
} else {
|
|
female++;
|
|
const data = readFileSync(`./pictures_base/profile/Frau_${female}.jpg`);
|
|
await storeProfilePicture(data, emailToDirName(u[0].email));
|
|
}
|
|
const data = readFileSync(`./pictures_base/logo/${i}.jpg`);
|
|
await storeCompanyLogo(data, emailToDirName(u[0].email));
|
|
}
|
|
|
|
//Corporate Listings
|
|
filePath = `./data/commercials.json`;
|
|
data = readFileSync(filePath, 'utf8');
|
|
const commercialJsonData = JSON.parse(data) as CommercialPropertyListing[]; // Erwartet ein Array von Objekten
|
|
for (let index = 0; index < commercialJsonData.length; index++) {
|
|
const commercial = commercialJsonData[index];
|
|
const id = commercial.id;
|
|
delete commercial.id;
|
|
const user = getRandomItem(generatedUserData);
|
|
commercial.imageOrder = getFilenames(id);
|
|
commercial.imagePath = emailToDirName(user.email);
|
|
const insertionDate = getRandomDateWithinLastYear();
|
|
commercial.created = insertionDate;
|
|
commercial.updated = insertionDate;
|
|
commercial.email = user.email;
|
|
commercial.draft = false;
|
|
commercial.type = sso.typesOfCommercialProperty.find(e => e.oldValue === String(commercial.type)).value;
|
|
const cityGeo = geos.states.find(s => s.state_code === commercial.state).cities.find(c => c.name === commercial.city);
|
|
try {
|
|
commercial.latitude = cityGeo.latitude;
|
|
commercial.longitude = cityGeo.longitude;
|
|
} catch (e) {
|
|
console.log(`----------------> ERROR ${commercial.state} - ${commercial.city}`);
|
|
}
|
|
// const reducedCommercial = {
|
|
// city: commercial.city,
|
|
// description: commercial.description,
|
|
// email: commercial.email,
|
|
// price: commercial.price,
|
|
// state: sso.locations.find(l => l.value === commercial.state)?.name,
|
|
// title: commercial.title,
|
|
// name: `${user.firstname} ${user.lastname}`,
|
|
// };
|
|
// const embedding = await createEmbedding(JSON.stringify(reducedCommercial));
|
|
// sleep(200);
|
|
const result = await db.insert(schema.commercials).values(commercial).returning();
|
|
// logger.info(`commercial_${index} inserted`);
|
|
try {
|
|
fs.copySync(`./pictures_base/property/${id}`, `./pictures/property/${result[0].imagePath}/${result[0].serialId}`);
|
|
} catch (err) {
|
|
console.log(`----- No pictures available for ${id} ------`);
|
|
}
|
|
}
|
|
|
|
//Business Listings
|
|
filePath = `./data/businesses.json`;
|
|
data = readFileSync(filePath, 'utf8');
|
|
const businessJsonData = JSON.parse(data) as BusinessListing[]; // Erwartet ein Array von Objekten
|
|
for (let index = 0; index < businessJsonData.length; index++) {
|
|
const business = businessJsonData[index];
|
|
delete business.id;
|
|
business.type = sso.typesOfBusiness.find(e => e.oldValue === String(business.type)).value;
|
|
business.created = new Date(business.created);
|
|
business.updated = new Date(business.created);
|
|
const user = getRandomItem(generatedUserData);
|
|
business.email = user.email;
|
|
business.imageName = emailToDirName(user.email);
|
|
const cityGeo = geos.states.find(s => s.state_code === business.state).cities.find(c => c.name === business.city);
|
|
try {
|
|
business.latitude = cityGeo.latitude;
|
|
business.longitude = cityGeo.longitude;
|
|
} catch (e) {
|
|
console.log(`----------------> ERROR ${business.state} - ${business.city}`);
|
|
}
|
|
// const embeddingText = JSON.stringify({
|
|
// type: typesOfBusiness.find(b => b.value === String(business.type))?.name,
|
|
// title: business.title,
|
|
// description: business.description,
|
|
// email: business.email,
|
|
// city: business.city,
|
|
// state: sso.locations.find(l => l.value === business.state)?.name,
|
|
// price: business.price,
|
|
// realEstateIncluded: business.realEstateIncluded,
|
|
// leasedLocation: business.leasedLocation,
|
|
// franchiseResale: business.franchiseResale,
|
|
// salesRevenue: business.salesRevenue,
|
|
// cashFlow: business.cashFlow,
|
|
// supportAndTraining: business.supportAndTraining,
|
|
// employees: business.employees,
|
|
// established: business.established,
|
|
// reasonForSale: business.reasonForSale,
|
|
// name: `${user.firstname} ${user.lastname}`,
|
|
// });
|
|
// const embedding = await createEmbedding(embeddingText);
|
|
sleep(200);
|
|
await db.insert(schema.businesses).values(business);
|
|
}
|
|
|
|
//End
|
|
await client.end();
|
|
|
|
function sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
async function createEmbedding(text: string): Promise<number[]> {
|
|
const response = await openai.embeddings.create({
|
|
model: 'text-embedding-3-small',
|
|
input: text,
|
|
});
|
|
return response.data[0].embedding;
|
|
}
|
|
|
|
function getRandomItem<T>(arr: T[]): T {
|
|
if (arr.length === 0) {
|
|
throw new Error('The array is empty.');
|
|
}
|
|
|
|
const randomIndex = Math.floor(Math.random() * arr.length);
|
|
return arr[randomIndex];
|
|
}
|
|
function getFilenames(id: string): string[] {
|
|
try {
|
|
let filePath = `./pictures_base/property/${id}`;
|
|
return readdirSync(filePath);
|
|
} catch (e) {
|
|
return [];
|
|
}
|
|
}
|
|
function getRandomDateWithinLastYear(): Date {
|
|
const currentDate = new Date();
|
|
const lastYear = new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate());
|
|
|
|
const timeDiff = currentDate.getTime() - lastYear.getTime();
|
|
const randomTimeDiff = Math.random() * timeDiff;
|
|
const randomDate = new Date(lastYear.getTime() + randomTimeDiff);
|
|
|
|
return randomDate;
|
|
}
|
|
async function storeProfilePicture(buffer: Buffer, userId: string) {
|
|
let quality = 50;
|
|
const output = await sharp(buffer)
|
|
.resize({ width: 300 })
|
|
.avif({ quality }) // Verwende AVIF
|
|
//.webp({ quality }) // Verwende Webp
|
|
.toBuffer();
|
|
await sharp(output).toFile(`./pictures/profile/${userId}.avif`);
|
|
}
|
|
|
|
async function storeCompanyLogo(buffer: Buffer, adjustedEmail: string) {
|
|
let quality = 50;
|
|
const output = await sharp(buffer)
|
|
.resize({ width: 300 })
|
|
.avif({ quality }) // Verwende AVIF
|
|
//.webp({ quality }) // Verwende Webp
|
|
.toBuffer();
|
|
await sharp(output).toFile(`./pictures/logo/${adjustedEmail}.avif`); // Ersetze Dateierweiterung
|
|
// await fs.outputFile(`./pictures/logo/${userId}`, file.buffer);
|
|
}
|
|
|
|
function deleteFilesOfDir(directoryPath) {
|
|
// Überprüfen, ob das Verzeichnis existiert
|
|
if (existsSync(directoryPath)) {
|
|
// Den Inhalt des Verzeichnisses synchron löschen
|
|
try {
|
|
readdirSync(directoryPath).forEach(file => {
|
|
const filePath = join(directoryPath, file);
|
|
// Wenn es sich um ein Verzeichnis handelt, rekursiv löschen
|
|
if (statSync(filePath).isDirectory()) {
|
|
rimraf.sync(filePath);
|
|
} else {
|
|
// Wenn es sich um eine Datei handelt, direkt löschen
|
|
unlinkSync(filePath);
|
|
}
|
|
});
|
|
console.log('Der Inhalt des Verzeichnisses wurde erfolgreich gelöscht.');
|
|
} catch (err) {
|
|
console.error('Fehler beim Löschen des Verzeichnisses:', err);
|
|
}
|
|
} else {
|
|
console.log('Das Verzeichnis existiert nicht.');
|
|
}
|
|
}
|