Greenlens/services/backend/openAiScanService.ts

565 lines
19 KiB
TypeScript

import { CareInfo, IdentificationResult, Language } from '../../types';
type OpenAiScanMode = 'primary' | 'review';
export interface OpenAiHealthIssue {
title: string;
confidence: number;
details: string;
}
export interface OpenAiHealthAnalysis {
overallHealthScore: number;
status: 'healthy' | 'watch' | 'critical';
likelyIssues: OpenAiHealthIssue[];
actionsNow: string[];
plan7Days: string[];
}
const OPENAI_API_KEY = (process.env.EXPO_PUBLIC_OPENAI_API_KEY || '').trim();
const OPENAI_SCAN_MODEL = (process.env.EXPO_PUBLIC_OPENAI_SCAN_MODEL || 'gpt-5-mini').trim();
const OPENAI_SCAN_MODEL_PRO = (process.env.EXPO_PUBLIC_OPENAI_SCAN_MODEL_PRO || OPENAI_SCAN_MODEL).trim();
const OPENAI_HEALTH_MODEL = (process.env.EXPO_PUBLIC_OPENAI_HEALTH_MODEL || OPENAI_SCAN_MODEL).trim();
const OPENAI_SCAN_FALLBACK_MODELS = (process.env.EXPO_PUBLIC_OPENAI_SCAN_FALLBACK_MODELS || 'gpt-5-mini,gpt-4.1-mini').trim();
const OPENAI_SCAN_FALLBACK_MODELS_PRO = (process.env.EXPO_PUBLIC_OPENAI_SCAN_FALLBACK_MODELS_PRO || OPENAI_SCAN_FALLBACK_MODELS).trim();
const OPENAI_HEALTH_FALLBACK_MODELS = (process.env.EXPO_PUBLIC_OPENAI_HEALTH_FALLBACK_MODELS || OPENAI_SCAN_FALLBACK_MODELS).trim();
const OPENAI_CHAT_COMPLETIONS_URL = 'https://api.openai.com/v1/chat/completions';
const OPENAI_TIMEOUT_MS = (() => {
const raw = (process.env.EXPO_PUBLIC_OPENAI_TIMEOUT_MS || '45000').trim();
const parsed = Number.parseInt(raw, 10);
if (Number.isFinite(parsed) && parsed >= 10000) return parsed;
return 45000;
})();
const parseModelChain = (primaryModel: string, fallbackModels: string): string[] => {
const models = [primaryModel];
fallbackModels.split(',').forEach((model) => {
const normalized = model.trim();
if (normalized) models.push(normalized);
});
return [...new Set(models)];
};
const OPENAI_SCAN_MODEL_CHAIN = parseModelChain(OPENAI_SCAN_MODEL, OPENAI_SCAN_FALLBACK_MODELS);
const OPENAI_SCAN_MODEL_CHAIN_PRO = parseModelChain(OPENAI_SCAN_MODEL_PRO, OPENAI_SCAN_FALLBACK_MODELS_PRO);
const OPENAI_HEALTH_MODEL_CHAIN = parseModelChain(OPENAI_HEALTH_MODEL, OPENAI_HEALTH_FALLBACK_MODELS);
const getScanModelChain = (plan: 'free' | 'pro'): string[] => {
return plan === 'pro' ? OPENAI_SCAN_MODEL_CHAIN_PRO : OPENAI_SCAN_MODEL_CHAIN;
};
const clamp = (value: number, min: number, max: number): number => {
return Math.min(max, Math.max(min, value));
};
const toErrorMessage = (error: unknown): string => {
if (error instanceof Error) return error.message;
return String(error);
};
const summarizeImageUri = (imageUri: string): string => {
const trimmed = imageUri.trim();
if (!trimmed) return 'empty';
if (trimmed.startsWith('data:image')) return `data-uri(${Math.round(trimmed.length / 1024)}kb)`;
return trimmed.length > 120 ? `${trimmed.slice(0, 120)}...` : trimmed;
};
const toJsonString = (content: string): string => {
const trimmed = content.trim();
if (!trimmed) return trimmed;
const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
if (fenced?.[1]) return fenced[1].trim();
return trimmed;
};
const parseContentToJson = (content: string): Record<string, unknown> | null => {
try {
const parsed = JSON.parse(toJsonString(content));
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
return null;
} catch {
return null;
}
};
const getString = (value: unknown): string => {
return typeof value === 'string' ? value.trim() : '';
};
const getNumber = (value: unknown): number | null => {
if (typeof value === 'number' && Number.isFinite(value)) return value;
if (typeof value === 'string' && value.trim()) {
const parsed = Number(value);
if (Number.isFinite(parsed)) return parsed;
}
return null;
};
const getStringArray = (value: unknown): string[] => {
if (!Array.isArray(value)) return [];
return value
.map((item) => (typeof item === 'string' ? item.trim() : ''))
.filter(Boolean);
};
const normalizeResult = (
raw: Record<string, unknown>,
language: Language,
): IdentificationResult | null => {
const name = getString(raw.name);
const botanicalName = getString(raw.botanicalName);
const description = getString(raw.description);
const confidenceRaw = getNumber(raw.confidence);
const careInfoRaw = raw.careInfo;
if (!name || !botanicalName || !careInfoRaw || typeof careInfoRaw !== 'object' || Array.isArray(careInfoRaw)) {
return null;
}
const careInfoObj = careInfoRaw as Record<string, unknown>;
const waterIntervalRaw = getNumber(careInfoObj.waterIntervalDays);
const light = getString(careInfoObj.light);
const temp = getString(careInfoObj.temp);
if (waterIntervalRaw == null || !light || !temp) {
return null;
}
const fallbackDescription = language === 'de'
? `${name} wurde per KI erkannt. Pflegehinweise sind unten aufgefuehrt.`
: language === 'es'
? `${name} se detecto con IA. Debajo veras recomendaciones de cuidado.`
: `${name} was identified with AI. Care guidance is shown below.`;
return {
name,
botanicalName,
confidence: clamp(confidenceRaw ?? 0.72, 0.05, 0.99),
description: description || fallbackDescription,
careInfo: {
waterIntervalDays: Math.round(clamp(waterIntervalRaw, 1, 45)),
light,
temp,
},
};
};
const getLanguageLabel = (language: Language): string => {
if (language === 'de') return 'German';
if (language === 'es') return 'Spanish';
return 'English';
};
const buildPrompt = (language: Language, mode: OpenAiScanMode): string => {
const reviewInstruction = mode === 'review'
? 'Re-check your first hypothesis with stricter botanical accuracy and correct any mismatch.'
: 'Identify the most likely houseplant species from this image with conservative confidence.';
const nameLanguageInstruction = language === 'en'
? '- "name" must be an English common name only. Never return a German or other non-English common name. If no reliable English common name is known, use "botanicalName" as "name" instead of inventing or translating.'
: `- "name" must be strictly written in ${getLanguageLabel(language)}. If a reliable common name in that language is not known, use "botanicalName" as "name" instead of inventing a localized name.`;
return [
`${reviewInstruction}`,
`Return strict JSON only in this shape:`,
`{"name":"...","botanicalName":"...","confidence":0.0,"description":"...","careInfo":{"waterIntervalDays":7,"light":"...","temp":"..."}}`,
`Rules:`,
nameLanguageInstruction,
`- "description" and "careInfo.light" must be written in ${getLanguageLabel(language)}.`,
`- "botanicalName" must use accepted Latin scientific naming and must not be invented or misspelled.`,
`- If species is uncertain, prefer genus-level naming (for example: "Calathea sp.").`,
`- "confidence" must be between 0 and 1.`,
`- Keep confidence <= 0.55 when the image is ambiguous, blurred, or partially visible.`,
`- "waterIntervalDays" must be an integer between 1 and 45.`,
`- Do not include markdown, explanations, or extra keys.`,
].join('\n');
};
const buildHealthPrompt = (
language: Language,
plantContext?: {
name: string;
botanicalName: string;
careInfo: CareInfo;
description?: string;
},
): string => {
const contextLines = plantContext
? [
`Plant context:`,
`- name: ${plantContext.name}`,
`- botanicalName: ${plantContext.botanicalName}`,
`- care.light: ${plantContext.careInfo.light}`,
`- care.temp: ${plantContext.careInfo.temp}`,
`- care.waterIntervalDays: ${plantContext.careInfo.waterIntervalDays}`,
`- description: ${plantContext.description || 'n/a'}`,
]
: ['Plant context: not provided'];
return [
`Analyze this plant photo for real health condition signs with focus on yellowing leaves, watering stress, pests, and light stress.`,
`Return strict JSON only in this shape:`,
`{"overallHealthScore":72,"status":"watch","likelyIssues":[{"title":"...","confidence":0.64,"details":"..."}],"actionsNow":["..."],"plan7Days":["..."]}`,
`Rules:`,
`- "overallHealthScore" must be an integer between 0 and 100.`,
`- "status" must be one of: "healthy", "watch", "critical".`,
`- "likelyIssues" must contain 1 to 4 items sorted by confidence descending.`,
`- "confidence" must be between 0 and 1.`,
`- "title", "details", "actionsNow", and "plan7Days" must be written in ${getLanguageLabel(language)}.`,
`- "actionsNow" should be immediate steps for the next 24 hours.`,
`- "plan7Days" should be short actionable steps for the next week.`,
`- Do not include markdown, explanations, or extra keys.`,
...contextLines,
].join('\n');
};
const buildFallbackHealthAnalysis = (
language: Language,
plantContext?: {
name: string;
botanicalName: string;
careInfo: CareInfo;
description?: string;
},
): OpenAiHealthAnalysis => {
if (language === 'de') {
return {
overallHealthScore: 58,
status: 'watch',
likelyIssues: [
{
title: 'Eingeschraenkte KI-Analyse',
confidence: 0.42,
details: `${plantContext?.name || 'Die Pflanze'} konnte wegen instabiler Antwort nicht vollstaendig bewertet werden.`,
},
],
actionsNow: [
'Neues Foto bei hellem, indirektem Licht aufnehmen.',
'Blaetter auf Flecken, Schaedlinge und trockene Raender pruefen.',
'Erst giessen, wenn die oberen 2-3 cm Erde trocken sind.',
],
plan7Days: [
'In 2 Tagen mit neuem Foto erneut pruefen.',
'Farbe und Blattspannung taeglich beobachten.',
'Bei Verschlechterung Standort und Giessrhythmus anpassen.',
],
};
}
if (language === 'es') {
return {
overallHealthScore: 58,
status: 'watch',
likelyIssues: [
{
title: 'Analisis de IA limitado',
confidence: 0.42,
details: `${plantContext?.name || 'La planta'} no pudo evaluarse por completo por una respuesta inestable.`,
},
],
actionsNow: [
'Tomar una foto nueva con luz brillante e indirecta.',
'Revisar hojas por manchas, plagas y bordes secos.',
'Regar solo si los 2-3 cm superiores del sustrato estan secos.',
],
plan7Days: [
'Revisar otra vez en 2 dias con una foto nueva.',
'Observar color y firmeza de hojas cada dia.',
'Si empeora, ajustar ubicacion y frecuencia de riego.',
],
};
}
return {
overallHealthScore: 58,
status: 'watch',
likelyIssues: [
{
title: 'Limited AI analysis',
confidence: 0.42,
details: `${plantContext?.name || 'This plant'} could not be fully assessed due to an unstable provider response.`,
},
],
actionsNow: [
'Capture a new photo in bright indirect light.',
'Inspect leaves for spots, pests, and dry edges.',
'Water only if the top 2-3 cm of soil is dry.',
],
plan7Days: [
'Re-check in 2 days with a new photo.',
'Track leaf color and firmness daily.',
'If symptoms worsen, adjust placement and watering cadence.',
],
};
};
const normalizeHealthAnalysis = (
raw: Record<string, unknown>,
language: Language,
): OpenAiHealthAnalysis | null => {
const scoreRaw = getNumber(raw.overallHealthScore);
const statusRaw = getString(raw.status);
const issuesRaw = raw.likelyIssues;
const actionsNowRaw = getStringArray(raw.actionsNow).slice(0, 6);
const plan7DaysRaw = getStringArray(raw.plan7Days).slice(0, 7);
if (scoreRaw == null || !statusRaw || !Array.isArray(issuesRaw)) {
return null;
}
const status: OpenAiHealthAnalysis['status'] = statusRaw === 'healthy' || statusRaw === 'watch' || statusRaw === 'critical'
? statusRaw
: 'watch';
const likelyIssues = issuesRaw
.map((entry) => {
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return null;
const issueObj = entry as Record<string, unknown>;
const title = getString(issueObj.title);
const details = getString(issueObj.details);
const confidenceRaw = getNumber(issueObj.confidence);
if (!title || !details || confidenceRaw == null) return null;
return {
title,
details,
confidence: clamp(confidenceRaw, 0.05, 0.99),
} as OpenAiHealthIssue;
})
.filter((entry): entry is OpenAiHealthIssue => Boolean(entry))
.slice(0, 4);
if (likelyIssues.length === 0 || actionsNowRaw.length === 0 || plan7DaysRaw.length === 0) {
const fallbackIssue = language === 'de'
? 'Die KI konnte keine stabilen Gesundheitsmerkmale extrahieren.'
: language === 'es'
? 'La IA no pudo extraer senales de salud estables.'
: 'AI could not extract stable health signals.';
return {
overallHealthScore: Math.round(clamp(scoreRaw, 0, 100)),
status,
likelyIssues: [
{
title: language === 'de'
? 'Analyse unsicher'
: language === 'es'
? 'Analisis incierto'
: 'Uncertain analysis',
confidence: 0.35,
details: fallbackIssue,
},
],
actionsNow: actionsNowRaw.length > 0
? actionsNowRaw
: [language === 'de' ? 'Neues, schaerferes Foto aufnehmen.' : language === 'es' ? 'Tomar una foto nueva y mas nitida.' : 'Capture a new, sharper photo.'],
plan7Days: plan7DaysRaw.length > 0
? plan7DaysRaw
: [language === 'de' ? 'In 2 Tagen erneut pruefen.' : language === 'es' ? 'Volver a revisar en 2 dias.' : 'Re-check in 2 days.'],
};
}
return {
overallHealthScore: Math.round(clamp(scoreRaw, 0, 100)),
status,
likelyIssues,
actionsNow: actionsNowRaw,
plan7Days: plan7DaysRaw,
};
};
const extractMessageContent = (payload: unknown): string => {
const response = payload as {
choices?: Array<{
message?: {
content?: string | Array<{ type?: string; text?: string }>;
};
}>;
};
const content = response.choices?.[0]?.message?.content;
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return content
.map((chunk) => (chunk?.type === 'text' ? chunk.text || '' : ''))
.join('')
.trim();
}
return '';
};
const postChatCompletion = async (
modelChain: string[],
imageUri: string,
messages: Array<Record<string, unknown>>,
): Promise<{ payload: Record<string, unknown> | null; modelUsed: string | null; attemptedModels: string[] }> => {
const attemptedModels: string[] = [];
for (const model of modelChain) {
attemptedModels.push(model);
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), OPENAI_TIMEOUT_MS);
try {
const response = await fetch(OPENAI_CHAT_COMPLETIONS_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${OPENAI_API_KEY}`,
},
body: JSON.stringify({
model,
response_format: { type: 'json_object' },
messages,
}),
signal: controller.signal,
});
if (!response.ok) {
const body = await response.text();
console.warn('OpenAI request HTTP error.', {
status: response.status,
model,
image: summarizeImageUri(imageUri),
bodyPreview: body.slice(0, 300),
});
continue;
}
const payload = (await response.json()) as Record<string, unknown>;
return { payload, modelUsed: model, attemptedModels };
} catch (error) {
const isTimeoutAbort = error instanceof Error && error.name === 'AbortError';
console.warn('OpenAI request failed.', {
model,
timeoutMs: OPENAI_TIMEOUT_MS,
aborted: isTimeoutAbort,
error: toErrorMessage(error),
image: summarizeImageUri(imageUri),
});
continue;
} finally {
clearTimeout(timeout);
}
}
return { payload: null, modelUsed: null, attemptedModels };
};
export const openAiScanService = {
isConfigured: (): boolean => Boolean(OPENAI_API_KEY),
identifyPlant: async (
imageUri: string,
language: Language,
mode: OpenAiScanMode = 'primary',
plan: 'free' | 'pro' = 'free',
): Promise<IdentificationResult | null> => {
if (!OPENAI_API_KEY) return null;
const modelChain = getScanModelChain(plan);
const completion = await postChatCompletion(
modelChain,
imageUri,
[
{
role: 'system',
content: 'You are a plant identification assistant. Return strict JSON only.',
},
{
role: 'user',
content: [
{ type: 'text', text: buildPrompt(language, mode) },
{ type: 'image_url', image_url: { url: imageUri } },
],
},
],
);
if (!completion.payload) return null;
const content = extractMessageContent(completion.payload);
if (!content) {
console.warn('OpenAI plant scan returned empty message content.', {
model: completion.modelUsed || modelChain[0],
mode,
image: summarizeImageUri(imageUri),
});
return null;
}
const parsed = parseContentToJson(content);
if (!parsed) {
console.warn('OpenAI plant scan returned non-JSON content.', {
model: completion.modelUsed || modelChain[0],
mode,
preview: content.slice(0, 220),
});
return null;
}
const normalized = normalizeResult(parsed, language);
if (!normalized) {
console.warn('OpenAI plant scan JSON did not match required schema.', {
model: completion.modelUsed || modelChain[0],
mode,
keys: Object.keys(parsed),
});
}
return normalized;
},
analyzePlantHealth: async (
imageUri: string,
language: Language,
plantContext?: {
name: string;
botanicalName: string;
careInfo: CareInfo;
description?: string;
},
): Promise<OpenAiHealthAnalysis | null> => {
if (!OPENAI_API_KEY) return null;
const completion = await postChatCompletion(
OPENAI_HEALTH_MODEL_CHAIN,
imageUri,
[
{
role: 'system',
content: 'You are a plant health diagnosis assistant. Return strict JSON only.',
},
{
role: 'user',
content: [
{ type: 'text', text: buildHealthPrompt(language, plantContext) },
{ type: 'image_url', image_url: { url: imageUri } },
],
},
],
);
if (!completion.payload) return buildFallbackHealthAnalysis(language, plantContext);
const content = extractMessageContent(completion.payload);
if (!content) {
console.warn('OpenAI health check returned empty content.', {
model: completion.modelUsed || OPENAI_HEALTH_MODEL_CHAIN[0],
image: summarizeImageUri(imageUri),
});
return buildFallbackHealthAnalysis(language, plantContext);
}
const parsed = parseContentToJson(content);
if (!parsed) {
console.warn('OpenAI health check returned non-JSON content.', {
model: completion.modelUsed || OPENAI_HEALTH_MODEL_CHAIN[0],
preview: content.slice(0, 220),
});
return buildFallbackHealthAnalysis(language, plantContext);
}
return normalizeHealthAnalysis(parsed, language) || buildFallbackHealthAnalysis(language, plantContext);
},
};