data & perftest

bizmatch-server/src/listings/business-listings.controller.ts

@@ -46,5 +46,9 @@ export class BusinessListingsController {
     this.listingsService.deleteBusinessListing(id)
   }
   
+  @Delete('deleteAll')
+  deleteAll(){
+    this.listingsService.deleteAllBusinessListings()
+  }
 }
 

bizmatch-server/src/listings/commercial-property-listings.controller.ts

@@ -43,6 +43,9 @@ export class CommercialPropertyListingsController {
     deleteById(@Param('id') id:string){
       this.listingsService.deleteCommercialPropertyListing(id)
     }
-    
+    @Delete('deleteAll')
+    deleteAll(){
+      this.listingsService.deleteAllcommercialListings()
+    }
 
 }

bizmatch-server/src/listings/listings.service.ts

@@ -4,7 +4,8 @@ import {
   CommercialPropertyListing,
   ListingCriteria,
   ListingType,
-  ImageProperty
+  ImageProperty,
+  ListingCategory
 } from '../models/main.model.js';
 import { convertStringToNullUndefined } from '../utils.js';
 import { WINSTON_MODULE_PROVIDER } from 'nest-winston';
@@ -14,6 +15,8 @@ import { REDIS_CLIENT } from '../redis/redis.module.js';
 
 @Injectable()
 export class ListingsService {
+  schemaNameBusiness:ListingCategory={name:'business'}
+  schemaNameCommercial:ListingCategory={name:'commercialProperty'}
   businessListingRepository:Repository;
   commercialPropertyListingRepository:Repository;
   baseListingSchemaDef : SchemaDefinition = {
@@ -53,13 +56,13 @@ export class ListingsService {
     ...this.baseListingSchemaDef,
     imageNames:{ type: 'string[]' },
   }
-  businessListingSchema = new Schema('businessListing',this.businessListingSchemaDef, {
+  businessListingSchema = new Schema(this.schemaNameBusiness.name,this.businessListingSchemaDef, {
       dataStructure: 'JSON'
   })
-  commercialPropertyListingSchema = new Schema('commercialPropertyListing',this.commercialPropertyListingSchemaDef, {
+  commercialPropertyListingSchema = new Schema(this.schemaNameCommercial.name,this.commercialPropertyListingSchemaDef, {
     dataStructure: 'JSON'
   })
-  constructor(@Inject(REDIS_CLIENT) private readonly redis: any){
+  constructor(@Inject(REDIS_CLIENT) private readonly redis: any, @Inject(WINSTON_MODULE_PROVIDER) private readonly logger: Logger){
     this.businessListingRepository = new Repository(this.businessListingSchema, redis);
     this.commercialPropertyListingRepository = new Repository(this.commercialPropertyListingSchema, redis)
     this.businessListingRepository.createIndex();
@@ -99,13 +102,40 @@ export class ListingsService {
     return await this.commercialPropertyListingRepository.search().return.all()
   }
   async findBusinessListings(criteria:ListingCriteria): Promise<any> {
-    let listings = await this.getAllBusinessListings();
-    return this.find(criteria,listings);
+    // let listings = await this.getAllBusinessListings();
+    // return this.find(criteria,listings);
+    this.logger.info(`start findBusinessListings: ${JSON.stringify(criteria)}`);
+    const result = await this.redis.ft.search('business:index','*',{LIMIT:{from:0,size:50}});
+    this.logger.info(`start findBusinessListings: ${JSON.stringify(criteria)}`);
+    return result.documents;
   }
   async findCommercialPropertyListings(criteria:ListingCriteria): Promise<any> {
     let listings = await this.getAllCommercialListings();
     return this.find(criteria,listings);
   }
+  async deleteAllBusinessListings(){
+    const ids = await this.getIdsForRepo(this.schemaNameBusiness.name);
+    this.businessListingRepository.remove(ids);
+  }
+  async deleteAllcommercialListings(){
+    const ids = await this.getIdsForRepo(this.schemaNameCommercial.name);
+    this.commercialPropertyListingRepository.remove(ids);
+  }
+  async getIdsForRepo(repoName:string, maxcount=100000){
+      let cursor = 0;
+      let ids = [];
+      do {
+        const reply = await this.redis.scan(cursor, {
+          MATCH: `${repoName}:*`,
+          COUNT: maxcount
+        });
+        cursor = reply.cursor;
+        // Extrahiere die ID aus jedem Schlüssel und füge sie zur Liste hinzu
+        ids = ids.concat(reply.keys.map(key => key.split(':')[1]).filter(id=>id!='index'));
+      } while (cursor !== 0);
+      return ids;
+  }
+
   async find(criteria:ListingCriteria, listings: any[]): Promise<any> {
     listings=listings.filter(l=>l.listingsCategory===criteria.listingsCategory);
     if (convertStringToNullUndefined(criteria.type)){

bizmatch-server/src/listings/unknown-listings.controller.ts

@@ -21,6 +21,9 @@ export class UnknownListingsController {
       return await this.listingsService.getCommercialPropertyListingById(id);
     }
   }
-
+  @Get('repo/:repo')
+  async getAllByRepo(@Param('repo') repo:string): Promise<any> {
+    return await this.listingsService.getIdsForRepo(repo);
+  }
 }
 

bizmatch-server/src/select-options/select-options.service.ts

@@ -38,7 +38,6 @@ export class SelectOptionsService {
     ];
     public listingCategories: Array<KeyValue> = [
       { name: 'Business', value: 'business' },
-      // { name: 'Professionals/Brokers Directory', value: 'professionals_brokers' },
       { name: 'Commercial Property', value: 'commercialProperty' },
     ]
     public categories: Array<KeyValueStyle> = [

common-models/src/main.model.ts

@@ -20,6 +20,9 @@ export type SelectOption<T = number> = {
 export type ImageType = {
     name:'propertyPicture'|'companyLogo'|'profile',upload:string,delete:string,
 }
+export type ListingCategory = {
+    name: 'business' | 'commercialProperty'
+}
 export interface Listing {
     id: string;
     userId: string;

crawler/importListing.ts

@@ -1,19 +1,31 @@
+import yargs from 'yargs'
+import { hideBin } from 'yargs/helpers'
 import fs from 'fs-extra';
 import { selectFiles } from './filechooser.js';
 import path from 'path';
 
+const argv = yargs(hideBin(process.argv)).argv
+
+
 (async () => {
     const result = await selectFiles({startingPath:'./data',directoryFilter: (directoryName)=>{
         return false;
       },fileFilter: (fileName) => {
         return /\.json$/gi.test(fileName);
       },pageSize:10,multi:false});
-      console.log(result['selectedFiles'][0]);
-      const file = result['selectedFiles'][0];
-    //   const extension = path.extname(file);
-    //   const basefileName = path.basename(file,extension);      
+    if (argv.delete){ // delete==business or commercialProperty
+      const ids = await fetch(`http://localhost:3000/bizmatch/listings/undefined/repo/${argv.delete}`,{
+        method: 'GET',
+        headers: { 'Content-Type': 'application/json' },
+      })
+      await fetch(`http://localhost:3000/bizmatch/listings/undefined/repo/${argv.delete}`,{
+        method: 'DELETE',
+        headers: { 'Content-Type': 'application/json' },
+      })        
+    }
+    console.log(result['selectedFiles'][0]);
+    const file = result['selectedFiles'][0];
     const listings = await fs.readJson(file);
-    //listings.forEach(element => {
     for (const listing of listings) {
         const type = listing.listingsCategory
         const response = await fetch(`http://localhost:3000/bizmatch/listings/${type}`, {

crawler/package.json

@@ -20,6 +20,8 @@
     "ioredis": "^5.3.2",
     "node-fetch": "^3.3.2",
     "puppeteer": "^22.1.0",
+    "redis": "^4.6.13",
+    "winston": "^3.13.0",
     "yargs": "^17.7.2"
   }
 }

crawler/perftest.ts

@@ -0,0 +1,24 @@
+import  {createLogger,transports, format}  from 'winston';
+import { createClient } from 'redis';
+const logger = createLogger({
+    transports: [
+      new transports.Console(),
+    ],
+    format: format.combine(
+        format.colorize(),
+        format.timestamp(),
+        format.printf(({ timestamp, level, message, service }) => {
+          return `[${timestamp}] ${service} ${level}: ${message}`;
+        })
+      ),
+});
+const redis = await createClient()
+.on('error', err => console.log('Redis Client Error', err))
+.connect();
+
+(async () => {
+    logger.info(`start findBusinessListings:`);
+    const result = await redis.ft.search('business:index','*',{LIMIT:{from:0,size:50}});
+    logger.info(`end findBusinessListings:`);
+    redis.disconnect();
+})();