aknuth
/
bizmatch-project
1
0
Fork 0
Code Issues 7 Pull Requests Packages Projects 5 Releases Wiki Activity

Umstellung postgres 2. part

This commit is contained in:
Andreas Knuth 2024-04-15 22:05:20 +02:00
parent 7d10080069
commit c4cdcf4505
17 changed files with 18327 additions and 1957 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bizmatch-server/REDIS_INDEX.txt
View File

@ -1 +0,0 @@
FT.CREATE listingsIndex ON JSON PREFIX 1 listings: SCHEMA $.location AS location TAG SORTABLE $.price AS price NUMERIC SORTABLE $.listingsCategory AS listingsCategory TAG SORTABLE $.type AS type TAG SORTABLE

12
bizmatch-server/src/drizzle/businesses_json.model.ts
View File

@ -1,12 +0,0 @@
import { jsonb, varchar } from 'drizzle-orm/pg-core';
import { pgTable, text, primaryKey } from 'drizzle-orm/pg-core';
export const businesses_json = pgTable('businesses_json', {
id: varchar('id', { length: 255 }).primaryKey(),
data: jsonb('data'),
});
export type BusinessesJson = {
id: string;
data: Record<string, any>;
};

2
bizmatch-server/src/drizzle/drizzle.module.ts
View File

@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
import { drizzle } from 'drizzle-orm/node-postgres';
import pkg from 'pg';
const { Pool } = pkg;
import * as schema from './businesses_json.model.js';
import * as schema from './schema.js';
import { ConfigService } from '@nestjs/config';
import { jsonb, varchar } from 'drizzle-orm/pg-core';
import { PG_CONNECTION } from './schema.js';

30
bizmatch-server/src/drizzle/schema.ts
View File

@ -1,14 +1,32 @@
import { integer, serial, text, pgTable } from 'drizzle-orm/pg-core';
import { relations } from 'drizzle-orm';
import { jsonb, varchar } from 'drizzle-orm/pg-core';
import { integer, serial, text, pgTable, timestamp, jsonb, varchar } from 'drizzle-orm/pg-core';
import { relations, sql } from 'drizzle-orm';
export const PG_CONNECTION = 'PG_CONNECTION';
export const businesses_json = pgTable('businesses_json', {
id: varchar('id', { length: 255 }).primaryKey(),
export const businesses_json = pgTable('businesses', {
id: varchar('id', { length: 255 }).primaryKey().default(sql`uuid_generate_v4()`),
data: jsonb('data'),
created: timestamp('created'),
updated: timestamp('updated'),
visits: integer('visits'),
last_visit: timestamp('last_visit'),
});
export const commercials_json = pgTable('commercials', {
id: varchar('id', { length: 255 }).primaryKey().default(sql`uuid_generate_v4()`),
data: jsonb('data'),
created: timestamp('created'),
updated: timestamp('updated'),
visits: integer('visits'),
last_visit: timestamp('last_visit'),
});
export const users = pgTable('users', {
id: varchar('id', { length: 255 }).primaryKey().default(sql`uuid_generate_v4()`),
data: jsonb('data'),
created: timestamp('created'),
updated: timestamp('updated'),
visits: integer('visits'),
last_visit: timestamp('last_visit'),
});
export type BusinessesJson = {
id: string;
data: Record<string, any>;

24
bizmatch-server/src/listings/business-listings.controller.ts
View File

@ -4,7 +4,8 @@ import { convertStringToNullUndefined } from '../utils.js';
import { ListingsService } from './listings.service.js';
import { WINSTON_MODULE_PROVIDER } from 'nest-winston';
import { Logger } from 'winston';
import { ListingCriteria } from 'src/models/main.model.js';
import { ListingCriteria } from '../models/main.model.js';
import { businesses_json } from '../drizzle/schema.js';
@Controller('listings/business')
export class BusinessListingsController {
@ -13,43 +14,34 @@ export class BusinessListingsController {
@Inject(WINSTON_MODULE_PROVIDER) private readonly logger: Logger) {
}
@Get()
findAll(): any {
this.logger.info(`start findAll Listing`);
return this.listingsService.findListings();
}
@Get(':id')
findById(@Param('id') id:string): any {
return this.listingsService.findById(id);
return this.listingsService.findById(id,businesses_json);
}
@Get('user/:userid')
findByUserId(@Param('userid') userid:string): any {
return this.listingsService.findByUserId(userid);
return this.listingsService.findByUserId(userid,businesses_json);
}
@Post('search')
find(@Body() criteria: ListingCriteria): any {
return this.listingsService.findByState(criteria.state);
return this.listingsService.findListingsByCriteria(criteria,businesses_json);
}
@Post()
create(@Body() listing: any){
this.logger.info(`Save Listing`);
this.listingsService.createListing(listing)
this.listingsService.createListing(listing,businesses_json)
}
@Put()
update(@Body() listing: any){
this.logger.info(`Save Listing`);
this.listingsService.updateListing(listing.id,listing)
this.listingsService.updateListing(listing.id,listing,businesses_json)
}
@Delete(':id')
deleteById(@Param('id') id:string){
this.listingsService.deleteListing(id)
this.listingsService.deleteListing(id,businesses_json)
}
// @Delete('deleteAll')
// deleteAll(){
// this.listingsService.deleteAllBusinessListings()
// }
}

69
bizmatch-server/src/listings/commercial-property-listings.controller.ts
View File

@ -4,7 +4,8 @@ import { WINSTON_MODULE_PROVIDER } from 'nest-winston';
import { Logger } from 'winston';
import { FileInterceptor } from '@nestjs/platform-express';
import { FileService } from '../file/file.service.js';
import { CommercialPropertyListing, ImageProperty } from 'src/models/main.model.js';
import { CommercialPropertyListing, ImageProperty, ListingCriteria } from '../models/main.model.js';
import { commercials_json } from '../drizzle/schema.js';
@Controller('listings/commercialProperty')
export class CommercialPropertyListingsController {
@ -12,40 +13,36 @@ export class CommercialPropertyListingsController {
constructor(private readonly listingsService:ListingsService,private fileService:FileService,@Inject(WINSTON_MODULE_PROVIDER) private readonly logger: Logger) {
}
// @Get(':id')
// findById(@Param('id') id:string): any {
// return this.listingsService.getCommercialPropertyListingById(id);
// }
// @Post('search')
// find(@Body() criteria: any): any {
// return this.listingsService.findCommercialPropertyListings(criteria);
// }
// @Put('imageOrder/:id')
// async changeImageOrder(@Param('id') id:string,@Body() imageOrder: ImageProperty[]) {
// this.listingsService.updateImageOrder(id, imageOrder)
// }
// /**
// * @param listing creates a new listing
// */
// @Post()
// save(@Body() listing: any){
// this.logger.info(`Save Listing`);
// this.listingsService.saveListing(listing)
// }
// /**
// * @param id deletes a listing
// */
// @Delete(':id')
// deleteById(@Param('id') id:string){
// this.listingsService.deleteCommercialPropertyListing(id)
// }
// @Delete('deleteAll')
// deleteAll(){
// this.listingsService.deleteAllcommercialListings()
// }
@Get(':id')
findById(@Param('id') id:string): any {
return this.listingsService.findById(id,commercials_json);
}
@Get('user/:userid')
findByUserId(@Param('userid') userid:string): any {
return this.listingsService.findByUserId(userid,commercials_json);
}
@Post('search')
find(@Body() criteria: ListingCriteria): any {
return this.listingsService.findByState(criteria.state,commercials_json);
}
@Post()
create(@Body() listing: any){
this.logger.info(`Save Listing`);
this.listingsService.createListing(listing,commercials_json)
}
@Put()
update(@Body() listing: any){
this.logger.info(`Save Listing`);
this.listingsService.updateListing(listing.id,listing,commercials_json)
}
@Delete(':id')
deleteById(@Param('id') id:string){
this.listingsService.deleteListing(id,commercials_json)
}
@Put('imageOrder/:id')
async changeImageOrder(@Param('id') id:string,@Body() imageOrder: ImageProperty[]) {
this.listingsService.updateImageOrder(id, imageOrder)
}
}

1
bizmatch-server/src/listings/listings.module.ts
View File

@ -10,7 +10,6 @@ import { BrokerListingsController } from './broker-listings.controller.js';
import { UserService } from '../user/user.service.js';
import { Client, Connection } from 'pg';
import { drizzle } from 'drizzle-orm/node-postgres';
import { businesses_json } from '../drizzle/businesses_json.model.js';
import { DrizzleModule } from '../drizzle/drizzle.module.js';

155
bizmatch-server/src/listings/listings.service.ts
View File

@ -5,85 +5,113 @@ import {
ListingCriteria,
ListingType,
ImageProperty,
ListingCategory
ListingCategory,
ResponseBusinessListing
} from '../models/main.model.js';
import { convertStringToNullUndefined } from '../utils.js';
import { WINSTON_MODULE_PROVIDER } from 'nest-winston';
import { Logger } from 'winston';
import { EntityData, EntityId, Schema, SchemaDefinition } from 'redis-om';
import { eq, ilike, sql } from 'drizzle-orm';
import { BusinessesJson, PG_CONNECTION, businesses_json } from '../drizzle/schema.js';
import { SQL, eq, ilike, sql } from 'drizzle-orm';
import { BusinessesJson, PG_CONNECTION, businesses_json, commercials_json } from '../drizzle/schema.js';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import * as schema from '../drizzle/schema.js';
import { PgTableFn, PgTableWithColumns, QueryBuilder } from 'drizzle-orm/pg-core';
@Injectable()
export class ListingsService {
constructor(@Inject(WINSTON_MODULE_PROVIDER) private readonly logger: Logger,
@Inject(PG_CONNECTION) private conn: NodePgDatabase<typeof schema>,){
// this.businessListingRepository = new Repository(this.businessListingSchema, redis);
// this.commercialPropertyListingRepository = new Repository(this.commercialPropertyListingSchema, redis)
// this.businessListingRepository.createIndex();
// this.commercialPropertyListingRepository.createIndex();
@Inject(PG_CONNECTION) private conn: NodePgDatabase<typeof schema>,) {
}
private buildWhereClause(criteria: ListingCriteria):SQL {
const finalSql = sql`1=1`;
finalSql.append(criteria.type?sql` AND data->>'type' = ${criteria.type}` : sql``)
finalSql.append(criteria.state ? sql` AND data->>'state' = ${criteria.state}` : sql``)
finalSql.append(criteria.minPrice ? sql` AND CAST(data->>'price' AS NUMERIC) >= ${parseFloat(criteria.minPrice)}` : sql``)
finalSql.append(criteria.maxPrice ? sql` AND CAST(data->>'price' AS NUMERIC) < ${parseFloat(criteria.maxPrice)}` : sql``)
finalSql.append(criteria.realEstateChecked !== undefined ? sql` AND CAST(data->>'realEstateIncluded' AS BOOLEAN) = ${criteria.realEstateChecked}` : sql``)
finalSql.append(criteria.title ? sql` AND LOWER(data->>'title') LIKE LOWER('%' || ${criteria.title} || '%')` : sql``)
return finalSql
}
// ##############################################################
// Listings general
// ##############################################################
private async findListings(table: typeof businesses_json | typeof commercials_json, criteria: ListingCriteria ,start = 0, length = 12): Promise<any> {
const whereClause = this.buildWhereClause(criteria)
const [data, total] = await Promise.all([
(await this.conn.select({ id:table.id, data: table.data }).from(table).where(whereClause).offset(start).limit(length)).map(e=>{
const ret = e.data as any
ret.id = e.id
return ret
}),
this.conn.select({ count: sql`count(*)` }).from(table).where(whereClause).then((result) => Number(result[0].count)),
]);
return { total, data };
}
async findById(id: string, table: typeof businesses_json | typeof commercials_json): Promise<BusinessListing|CommercialPropertyListing> {
const result = await this.conn.select({ data: table.data }).from(table).where(sql`${table.id} = ${id}`)
return result[0].data as BusinessListing
}
// ##############################################################
// ##############################################################
async createListing(newListing: { id: string; data: BusinessesJson }): Promise<BusinessesJson> {
const [createdListing] = await this.conn.insert(businesses_json).values(newListing).returning();
async findListingsByCriteria(criteria: ListingCriteria, table: typeof businesses_json | typeof commercials_json): Promise<{ data: Record<string, any>[]; total: number }> {
const start = criteria.start ? criteria.start : 0;
const length = criteria.length ? criteria.length : 12;
return await this.findListings(table, criteria, start, length)
}
async findByPriceRange(minPrice: number, maxPrice: number, table: typeof businesses_json | typeof commercials_json): Promise<BusinessesJson[]> {
return this.conn.select().from(table).where(sql`${table.data}->>'price' BETWEEN ${minPrice} AND ${maxPrice}`);
}
async findByState(state: string, table: typeof businesses_json | typeof commercials_json): Promise<BusinessesJson[]> {
return this.conn.select().from(table).where(sql`${table.data}->>'state' = ${state}`);
}
async findByUserId(userId: string, table: typeof businesses_json | typeof commercials_json): Promise<BusinessesJson[]> {
return this.conn.select().from(table).where(sql`${table.data}->>'userId' = ${userId}`);
}
async findByTitleContains(title: string, table: typeof businesses_json | typeof commercials_json): Promise<BusinessesJson[]> {
return this.conn.select().from(table).where(sql`${table.data}->>'title' ILIKE '%' || ${title} || '%'`);
}
async createListing(data: BusinessListing, table: typeof businesses_json | typeof commercials_json): Promise<BusinessesJson> {
const newListing = { data, created: data.created, updated: data.updated, visits: 0, last_visit: null }
const [createdListing] = await this.conn.insert(table).values(newListing).returning();
return createdListing as BusinessesJson;
}
async updateListing(id: string, data: BusinessListing): Promise<BusinessesJson> {
const [updateListing] = await this.conn.update(businesses_json).set(data).where(eq(businesses_json.id, id)).returning();
async updateListing(id: string, data: BusinessListing | CommercialPropertyListing, table: typeof businesses_json | typeof commercials_json): Promise<BusinessesJson> {
const [updateListing] = await this.conn.update(table).set(data).where(eq(table.id, id)).returning();
return updateListing as BusinessesJson;
}
async deleteListing(id: string): Promise<void> {
await this.conn.delete(businesses_json).where(eq(businesses_json.id, id));
async deleteListing(id: string, table: typeof businesses_json | typeof commercials_json): Promise<void> {
await this.conn.delete(table).where(eq(table.id, id));
}
async findByPriceRange(minPrice: number, maxPrice: number): Promise<BusinessesJson[]> {
return this.conn.select().from(businesses_json).where(sql`${businesses_json.data}->>'price' BETWEEN ${minPrice} AND ${maxPrice}`);
}
// ##############################################################
// Images for commercial Properties
// ##############################################################
async findByState(state: string): Promise<BusinessesJson[]> {
return this.conn.select().from(businesses_json).where(sql`${businesses_json.data}->>'state' = ${state}`);
async updateImageOrder(id: string, imageOrder: ImageProperty[]) {
const listing = await this.findById(id, commercials_json) as unknown as CommercialPropertyListing
listing.imageOrder = imageOrder;
await this.updateListing(listing.id, listing, commercials_json)
}
async findById(id: string): Promise<BusinessesJson[]> {
return this.conn.select().from(businesses_json).where(sql`${businesses_json.id} = ${id}`);
async deleteImage(id: string, name: string,) {
const listing = await this.findById(id, commercials_json) as unknown as CommercialPropertyListing
const index = listing.imageOrder.findIndex(im => im.name === name);
if (index > -1) {
listing.imageOrder.splice(index, 1);
await this.updateListing(listing.id, listing, commercials_json)
}
}
async findByUserId(userId: string): Promise<BusinessesJson[]> {
return this.conn.select().from(businesses_json).where(sql`${businesses_json.data}->>'userId' = ${userId}`);
async addImage(id: string, imagename: string) {
const listing = await this.findById(id, commercials_json) as unknown as CommercialPropertyListing
listing.imageOrder.push({ name: imagename, code: '', id: '' });
await this.updateListing(listing.id, listing, commercials_json)
}
// async findByTitleContains(title: string): Promise<BusinessesJson[]> {
// return this.conn.select().from(businesses_json).where(ilike(sql`${businesses_json.data}->>'title'`, `%${title}%`));
// }
async findByTitleContains(title: string): Promise<BusinessesJson[]> {
return this.conn.select().from(businesses_json).where(sql`${businesses_json.data}->>'title' ILIKE '%' || ${title} || '%'`);
}
async findListings(start = 0, size = 48): Promise<{ data: Record<string, any>[]; total: number }> {
// return this.conn.select({ data: businesses_json.data }).from(businesses_json).offset(start).limit(size);
const [data, total] = await Promise.all([
this.conn.select({ data: businesses_json.data }).from(businesses_json).offset(start).limit(size),
this.conn.select({ count: sql`count(*)` }).from(businesses_json).then((result) => Number(result[0].count)),
]);
return { data, total };
}
// ##############################################################
// ##############################################################
// async saveListing(listing: BusinessListing | CommercialPropertyListing) {
// const repo=listing.listingsCategory==='business'?this.businessListingRepository:this.commercialPropertyListingRepository;
// let result
// if (listing.id){
// result = await repo.save(listing.id,listing as any)
// } else {
// result = await repo.save(listing as any)
// listing.id=result[EntityId];
// result = await repo.save(listing.id,listing as any)
// }
// return result;
// }
// async getCommercialPropertyListingById(id: string): Promise<CommercialPropertyListing>{
// return await this.commercialPropertyListingRepository.fetch(id) as unknown as CommercialPropertyListing;
// }
@ -171,25 +199,8 @@ export class ListingsService {
// return listings
// }
// async updateImageOrder(id:string,imageOrder: ImageProperty[]){
// const listing = await this.getCommercialPropertyListingById(id) as unknown as CommercialPropertyListing
// listing.imageOrder=imageOrder;
// this.saveListing(listing);
// }
// async deleteImage(listingid:string,name:string,){
// const listing = await this.getCommercialPropertyListingById(listingid) as unknown as CommercialPropertyListing
// const index = listing.imageOrder.findIndex(im=>im.name===name);
// if (index>-1){
// listing.imageOrder.splice(index,1);
// this.saveListing(listing);
// }
// }
// async addImage(id:string,imagename: string){
// const listing = await this.getCommercialPropertyListingById(id) as unknown as CommercialPropertyListing
// listing.imageOrder.push({name:imagename,code:'',id:''});
// this.saveListing(listing);
// }
}

23
bizmatch-server/src/listings/unknown-listings.controller.ts
View File

@ -4,6 +4,7 @@ import { convertStringToNullUndefined } from '../utils.js';
import { ListingsService } from './listings.service.js';
import { WINSTON_MODULE_PROVIDER } from 'nest-winston';
import { Logger } from 'winston';
import { businesses_json, commercials_json } from '../drizzle/schema.js';
@Controller('listings/undefined')
export class UnknownListingsController {
@ -11,19 +12,15 @@ export class UnknownListingsController {
constructor(private readonly listingsService:ListingsService,@Inject(WINSTON_MODULE_PROVIDER) private readonly logger: Logger) {
}
@Get(':id')
async findById(@Param('id') id:string): Promise<any> {
const result = await this.listingsService.findById(id,businesses_json);
if (result){
return result
} else {
return await this.listingsService.findById(id,commercials_json);
}
}
// @Get(':id')
// async findById(@Param('id') id:string): Promise<any> {
// const result = await this.listingsService.getBusinessListingById(id);
// if (result.id){
// return result
// } else {
// return await this.listingsService.getCommercialPropertyListingById(id);
// }
// }
// @Get('repo/:repo')
// async getAllByRepo(@Param('repo') repo:string): Promise<any> {
// return await this.listingsService.getIdsForRepo(repo);
// }
}

9
bizmatch/src/app/services/listings.service.ts
View File

@ -2,7 +2,7 @@ import { HttpClient } from '@angular/common/http';
import { Injectable } from '@angular/core';
import { Observable, lastValueFrom } from 'rxjs';
import { environment } from '../../environments/environment';
import { BusinessListing, ImageProperty, ListingCriteria, ListingType } from '../../../../common-models/src/main.model';
import { BusinessListing, CommercialPropertyListing, ImageProperty, ListingCriteria, ListingType, ResponseBusinessListing, ResponseBusinessListingArray, ResponseCommercialPropertyListing, ResponseCommercialPropertyListingArray } from '../../../../common-models/src/main.model';
import onChange from 'on-change';
import { getSessionStorageHandler } from '../utils/utils';
@ -18,11 +18,12 @@ export class ListingsService {
// return this.http.get<ListingType[]>(`${this.apiBaseUrl}/bizmatch/business-listings`);
// }
async getListings(criteria:ListingCriteria):Promise<ListingType[]>{
const result = await lastValueFrom(this.http.post<ListingType[]>(`${this.apiBaseUrl}/bizmatch/listings/${criteria.listingsCategory}/search`,criteria));
return result;
const result = await lastValueFrom(this.http.post<ResponseBusinessListingArray|ResponseCommercialPropertyListingArray>(`${this.apiBaseUrl}/bizmatch/listings/${criteria.listingsCategory}/search`,criteria));
return result.data;
}
getListingById(id:string,listingsCategory?:'business'|'commercialProperty'):Observable<ListingType>{
return this.http.get<ListingType>(`${this.apiBaseUrl}/bizmatch/listings/${listingsCategory}/${id}`);
const result = this.http.get<ListingType>(`${this.apiBaseUrl}/bizmatch/listings/${listingsCategory}/${id}`);
return result;
}
getListingByUserId(userid:string):Promise<BusinessListing[]>{
return lastValueFrom(this.http.get<BusinessListing[]>(`${this.apiBaseUrl}/bizmatch/listings/business/user/${userid}`));

15
common-models/src/main.model.ts
View File

@ -67,6 +67,20 @@ export type ListingType =
| BusinessListing
| CommercialPropertyListing;
export type ResponseBusinessListingArray = {
data:BusinessListing[],
total:number
}
export type ResponseBusinessListing = {
data:BusinessListing
}
export type ResponseCommercialPropertyListingArray = {
data:CommercialPropertyListing[],
total:number
}
export type ResponseCommercialPropertyListing = {
data:CommercialPropertyListing
}
export interface ListingCriteria {
start:number,
length:number,
@ -77,6 +91,7 @@ export interface ListingCriteria {
minPrice:string,
maxPrice:string,
realEstateChecked:boolean,
title:string,
listingsCategory:'business'|'professionals_brokers'|'commercialProperty',
category:'professional|broker'
}

16
crawler/.vscode/launch.json vendored
View File

@ -16,18 +16,6 @@
"${workspaceFolder}/**/*.js"
]
},
{
"type": "node",
"request": "launch",
"name": "Import",
"skipFiles": [
"<node_internals>/**"
],
"program": "${workspaceFolder}/import.js",
"outFiles": [
"${workspaceFolder}/**/*.js"
]
},
{
"type": "node",
"request": "launch",
@ -44,11 +32,11 @@
{
"type": "node",
"request": "launch",
"name": "updateFields",
"name": "postgres_business_import",
"skipFiles": [
"<node_internals>/**"
],
"program": "${workspaceFolder}/updateFields.js",
"program": "${workspaceFolder}/build/crawler/postgres_business_import.js",
"outFiles": [
"${workspaceFolder}/**/*.js"
]

3290
crawler/data/businesses.json
View File

File diff suppressed because it is too large Load Diff

12494
crawler/data/businesses_.json
View File

File diff suppressed because it is too large Load Diff

3988
crawler/data/commercials.json
View File

File diff suppressed because it is too large Load Diff

153
crawler/postgres_business_import.ts
View File

@ -3,6 +3,7 @@ const { Pool } = pkg;
import fsextra from 'fs-extra';
const { fstat, readFileSync, writeJsonSync } = fsextra;
import { v4 as uuidv4 } from 'uuid';
import { CommercialPropertyListing, User } from '../common-models/src/main.model';
// PostgreSQL Verbindungskonfiguration
const pool = new Pool({
user: 'bizmatch',
@ -39,47 +40,119 @@ interface BusinessListing {
created: Date;
}
// Funktion zum Einlesen und Importieren von JSON-Daten
async function importJsonData(filePath: string): Promise<void> {
try {
const data: string = readFileSync(filePath, 'utf8');
const jsonData: BusinessListing[] = JSON.parse(data); // Erwartet ein Array von Objekten
const out: BusinessListing[] =[]
// Daten für jedes Listing in die Datenbank einfügen
for (const listing of jsonData) {
// const uuid = uuidv4();
// listing.id=uuid;
const values = [
listing.userId, listing.listingsCategory, listing.title, listing.description,
listing.type, listing.state, listing.city, listing.id, listing.price, listing.salesRevenue,
listing.leasedLocation, listing.established, listing.employees,
listing.reasonForSale, listing.supportAndTraining, listing.cashFlow, listing.brokerLicencing,
listing.internalListingNumber, listing.realEstateIncluded, listing.franchiseResale,
listing.draft, listing.internals, listing.created, new Date(), 0, null
];
const json_values = [
listing.id, listing
]
await pool.query(`INSERT INTO businesses
(user_id, listings_category, title, description, type, state, city, id, price, sales_revenue, leased_location,
established, employees, reason_for_sale, support_and_training, cash_flow, broker_licencing, internal_listing_number,
real_estate_included, franchise_resale, draft, internals, created, updated, visits, last_visit)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)`, values);
await pool.query('INSERT INTO businesses_json (id, data) VALUES ($1,$2)', json_values);
// out.push(listing);
}
writeJsonSync('./data/businesses_.json',out);
console.log('All data imported successfully.');
} catch (err) {
console.error('Error importing data:', err.message);
} finally {
// Schließen der Verbindung zum Pool
await pool.end();
async function importBusinesses() {
const filePath = './data/businesses.json'
const data: string = readFileSync(filePath, 'utf8');
const jsonData: BusinessListing[]|any = JSON.parse(data); // Erwartet ein Array von Objekten
await pool.query('drop table if exists businesses');
await pool.query(`CREATE TABLE businesses (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
created TIMESTAMP,
updated TIMESTAMP,
visits INTEGER,
last_visit TIMESTAMP,
data jsonb
);`);
for (const listing of jsonData) {
const created = listing.created
delete listing.created;
delete listing.id;
delete listing.temporary
const json_values = [
created, new Date(), 0, null, listing
]
await pool.query('INSERT INTO businesses (created, updated, visits, last_visit, data) VALUES ($1,$2,$3,$4,$5)', json_values);
}
console.log('All data imported successfully.');
}
async function importUser() {
const filePath = './data/broker.json'
const data: string = readFileSync(filePath, 'utf8');
const jsonData: User[] = JSON.parse(data); // Erwartet ein Array von Objekten
await pool.query('drop table if exists users');
await pool.query(`CREATE TABLE users (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
created TIMESTAMP,
updated TIMESTAMP,
visits INTEGER,
last_visit TIMESTAMP,
data jsonb
);`);
for (const user of jsonData) {
delete user.id;
user.hasCompanyLogo=false;
user.hasProfile=false;
const json_values = [
getRandomDateLastYear(), new Date(), 0, null, user
]
await pool.query('INSERT INTO users (created, updated, visits, last_visit, data) VALUES ($1,$2,$3,$4,$5)', json_values);
}
console.log('All data imported successfully.');
}
async function importCommercials() {
const filePath = './data/commercials.json'
const data: string = readFileSync(filePath, 'utf8');
const jsonData: CommercialPropertyListing[]|any = JSON.parse(data); // Erwartet ein Array von Objekten
await pool.query('drop table if exists commercials');
await pool.query(`CREATE TABLE commercials (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
created TIMESTAMP,
updated TIMESTAMP,
visits INTEGER,
last_visit TIMESTAMP,
data jsonb
);`);
for (const commercial of jsonData) {
commercial.hasImages=false;
commercial.imagePath=commercial.id;
delete commercial.id;
delete commercial.temporary;
const json_values = [
getRandomDateLastYear(), new Date(), 0, null, commercial
]
await pool.query('INSERT INTO commercials (created, updated, visits, last_visit, data) VALUES ($1,$2,$3,$4,$5)', json_values);
}
console.log('All data imported successfully.');
}
function idUpdate(jsonData) {
const out: BusinessListing[] = []
for (const listing of jsonData) {
const uuid = uuidv4();
listing.id = uuid;
out.push(listing);
}
writeJsonSync('./data/businesses_.json', out);
console.log('All data updated sucessfully.');
}
function getRandomDateLastYear(): Date {
const today = new Date();
const lastYear = new Date(today.getFullYear() - 1, today.getMonth(), today.getDate());
// Generiere eine zufällige Zahl zwischen 0 und der Anzahl der Millisekunden in einem Jahr
const randomTime = Math.random() * (today.getTime() - lastYear.getTime());
// Erstelle ein neues Datum basierend auf dieser zufälligen Zeit
const randomDate = new Date(lastYear.getTime() + randomTime);
return randomDate;
}
// Passen Sie den Dateipfad an Ihre spezifischen Bedürfnisse an
importJsonData('./data/businesses_.json');
try {
await importBusinesses();
await importUser();
await importCommercials();
} catch (err) {
console.error('Error importing data:', err.message);
} finally {
// Schließen der Verbindung zum Pool
await pool.end();
}

2
crawler/tsconfig.json
View File

@ -11,7 +11,7 @@
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "ES2017", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
"target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */