feat(vwb): Intégration UI-DETR-1 + Toggle mode Basique/Intelligent/Debug
- Toggle 3 modes dans le header: Basique (coords fixes), Intelligent (vision IA), Debug (overlay) - Service UI-DETR-1 pour détection d'éléments UI (510MB model, ~800ms/image) - API endpoints: /api/ui-detection/detect, /preload, /status, /find-element - Overlay des bboxes détectées en mode Debug (miniature + plein écran) - Clic sur élément détecté pour le sélectionner comme ancre - Document de vision produit: docs/VISION_RPA_INTELLIGENT.md - Configuration CORS étendue pour ports locaux Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
237
visual_workflow_builder/backend/api/ui_detection.py
Normal file
237
visual_workflow_builder/backend/api/ui_detection.py
Normal file
@@ -0,0 +1,237 @@
|
||||
"""
|
||||
API Blueprint pour la détection UI avec UI-DETR-1
|
||||
"""
|
||||
|
||||
from flask import Blueprint, request, jsonify
|
||||
from flask_cors import cross_origin
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
|
||||
ui_detection_bp = Blueprint('ui_detection', __name__, url_prefix='/api/ui-detection')
|
||||
|
||||
# Import lazy du service (le modèle est lourd)
|
||||
_service = None
|
||||
|
||||
|
||||
def get_service():
|
||||
"""Lazy loading du service de détection"""
|
||||
global _service
|
||||
if _service is None:
|
||||
from services.ui_detection_service import (
|
||||
detect_from_base64,
|
||||
detect_from_file,
|
||||
annotated_image_to_base64,
|
||||
preload_model
|
||||
)
|
||||
_service = {
|
||||
'detect_from_base64': detect_from_base64,
|
||||
'detect_from_file': detect_from_file,
|
||||
'annotated_image_to_base64': annotated_image_to_base64,
|
||||
'preload_model': preload_model
|
||||
}
|
||||
return _service
|
||||
|
||||
|
||||
@ui_detection_bp.route('/detect', methods=['POST'])
|
||||
@cross_origin()
|
||||
def detect_ui_elements():
|
||||
"""
|
||||
Détecte les éléments UI dans une image
|
||||
|
||||
Request body (JSON):
|
||||
- image_base64: Image encodée en base64 (requis)
|
||||
- threshold: Seuil de confiance (optionnel, défaut: 0.35)
|
||||
- annotate: Retourner l'image annotée (optionnel, défaut: false)
|
||||
- show_confidence: Afficher les scores sur l'image annotée (optionnel, défaut: false)
|
||||
|
||||
Response:
|
||||
- success: bool
|
||||
- result: {
|
||||
elements: [...],
|
||||
count: int,
|
||||
processing_time_ms: float,
|
||||
image_size: {width, height},
|
||||
model: str,
|
||||
annotated_image_base64?: str (si annotate=true)
|
||||
}
|
||||
"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
|
||||
if not data or 'image_base64' not in data:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': 'image_base64 est requis'
|
||||
}), 400
|
||||
|
||||
image_base64 = data['image_base64']
|
||||
threshold = data.get('threshold', 0.35)
|
||||
annotate = data.get('annotate', False)
|
||||
show_confidence = data.get('show_confidence', False)
|
||||
|
||||
# Valider le threshold
|
||||
threshold = max(0.1, min(1.0, float(threshold)))
|
||||
|
||||
service = get_service()
|
||||
|
||||
# Détecter les éléments
|
||||
result = service['detect_from_base64'](image_base64, threshold)
|
||||
response_data = result.to_dict()
|
||||
|
||||
# Générer l'image annotée si demandé
|
||||
if annotate:
|
||||
# Décoder l'image originale
|
||||
if ',' in image_base64:
|
||||
image_base64_clean = image_base64.split(',')[1]
|
||||
else:
|
||||
image_base64_clean = image_base64
|
||||
|
||||
image_bytes = base64.b64decode(image_base64_clean)
|
||||
image = Image.open(io.BytesIO(image_bytes))
|
||||
|
||||
# Créer l'image annotée
|
||||
annotated_b64 = service['annotated_image_to_base64'](
|
||||
image, result,
|
||||
show_ids=True,
|
||||
show_confidence=show_confidence
|
||||
)
|
||||
response_data['annotated_image_base64'] = f"data:image/png;base64,{annotated_b64}"
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'result': response_data
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}), 500
|
||||
|
||||
|
||||
@ui_detection_bp.route('/preload', methods=['POST'])
|
||||
@cross_origin()
|
||||
def preload_model():
|
||||
"""
|
||||
Précharge le modèle UI-DETR-1 en mémoire
|
||||
|
||||
Utile pour éviter la latence du premier appel
|
||||
"""
|
||||
try:
|
||||
service = get_service()
|
||||
service['preload_model']()
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'message': 'Modèle en cours de chargement'
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}), 500
|
||||
|
||||
|
||||
@ui_detection_bp.route('/status', methods=['GET'])
|
||||
@cross_origin()
|
||||
def get_status():
|
||||
"""
|
||||
Retourne le statut du service de détection
|
||||
"""
|
||||
try:
|
||||
from services.ui_detection_service import _model, MODEL_PATH
|
||||
import os
|
||||
|
||||
model_exists = os.path.exists(MODEL_PATH)
|
||||
model_loaded = _model is not None
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'status': {
|
||||
'model_path': MODEL_PATH,
|
||||
'model_exists': model_exists,
|
||||
'model_loaded': model_loaded,
|
||||
'model_name': 'UI-DETR-1',
|
||||
'default_threshold': 0.35
|
||||
}
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}), 500
|
||||
|
||||
|
||||
@ui_detection_bp.route('/find-element', methods=['POST'])
|
||||
@cross_origin()
|
||||
def find_element():
|
||||
"""
|
||||
Trouve un élément spécifique dans l'image en utilisant une ancre de référence
|
||||
|
||||
Request body (JSON):
|
||||
- image_base64: Screenshot actuel
|
||||
- anchor_base64: Image de l'ancre à trouver
|
||||
- threshold: Seuil de confiance (optionnel)
|
||||
|
||||
Response:
|
||||
- success: bool
|
||||
- result: {
|
||||
found: bool,
|
||||
element: {...} ou null,
|
||||
all_elements: [...],
|
||||
match_score: float
|
||||
}
|
||||
|
||||
Note: Cette fonction utilise la détection + comparaison d'embedding CLIP
|
||||
"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
|
||||
if not data or 'image_base64' not in data:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': 'image_base64 est requis'
|
||||
}), 400
|
||||
|
||||
image_base64 = data['image_base64']
|
||||
anchor_base64 = data.get('anchor_base64')
|
||||
threshold = data.get('threshold', 0.35)
|
||||
|
||||
service = get_service()
|
||||
|
||||
# Détecter tous les éléments
|
||||
result = service['detect_from_base64'](image_base64, threshold)
|
||||
|
||||
response = {
|
||||
'found': False,
|
||||
'element': None,
|
||||
'all_elements': [e.to_dict() for e in result.elements],
|
||||
'count': len(result.elements),
|
||||
'match_score': 0.0
|
||||
}
|
||||
|
||||
# Si une ancre est fournie, essayer de la matcher
|
||||
if anchor_base64 and len(result.elements) > 0:
|
||||
# TODO: Intégrer CLIP pour le matching d'ancre
|
||||
# Pour l'instant, retourner le premier élément comme placeholder
|
||||
response['found'] = True
|
||||
response['element'] = result.elements[0].to_dict()
|
||||
response['match_score'] = 0.5 # Placeholder
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'result': response
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}), 500
|
||||
Reference in New Issue
Block a user