feat(coaching): Implement complete COACHING mode infrastructure
Add comprehensive COACHING mode system with: Backend: - core/coaching module with session persistence and metrics - CoachingSessionPersistence for pause/resume sessions - CoachingMetricsCollector with learning progress tracking - REST API blueprint for coaching sessions management - Execution integration with COACHING mode support Frontend: - CoachingPanel component with keyboard shortcuts - Decision buttons (accept/reject/correct/manual/skip) - Real-time stats display and correction editor - CorrectionPacksDashboard for pack visualization - WebSocket hooks for real-time COACHING events Metrics & Monitoring: - WorkflowLearningMetrics with confidence scoring - GlobalCoachingMetrics for system-wide analytics - AUTO mode readiness detection (85% acceptance threshold) - Learning progress levels (OBSERVATION → COACHING → AUTO) Tests: - E2E tests for complete OBSERVATION → AUTO journey - Session persistence and recovery tests - Metrics threshold validation tests Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* Hook for COACHING mode WebSocket communication.
|
||||
*
|
||||
* Manages real-time communication for COACHING sessions including:
|
||||
* - Subscribing to COACHING events
|
||||
* - Receiving action suggestions
|
||||
* - Submitting user decisions
|
||||
* - Tracking COACHING statistics
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import { io, Socket } from 'socket.io-client';
|
||||
|
||||
// Types for COACHING mode
|
||||
export type CoachingDecision = 'accept' | 'reject' | 'correct' | 'manual' | 'skip';
|
||||
|
||||
export interface CoachingSuggestion {
|
||||
executionId: string;
|
||||
action: string;
|
||||
target: Record<string, any>;
|
||||
params: Record<string, any>;
|
||||
confidence: number;
|
||||
alternatives?: Array<{
|
||||
action: string;
|
||||
target: Record<string, any>;
|
||||
confidence: number;
|
||||
}>;
|
||||
screenshotPath?: string;
|
||||
context?: Record<string, any>;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface CoachingStats {
|
||||
suggestionsMade: number;
|
||||
accepted: number;
|
||||
rejected: number;
|
||||
corrected: number;
|
||||
manualExecutions: number;
|
||||
acceptanceRate: number;
|
||||
correctionRate: number;
|
||||
}
|
||||
|
||||
export interface CoachingActionResult {
|
||||
executionId: string;
|
||||
action: string;
|
||||
success: boolean;
|
||||
result?: Record<string, any>;
|
||||
error?: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface UseCoachingWebSocketOptions {
|
||||
serverUrl?: string;
|
||||
autoConnect?: boolean;
|
||||
}
|
||||
|
||||
interface UseCoachingWebSocketReturn {
|
||||
isConnected: boolean;
|
||||
isSubscribed: boolean;
|
||||
currentSuggestion: CoachingSuggestion | null;
|
||||
stats: CoachingStats;
|
||||
lastActionResult: CoachingActionResult | null;
|
||||
error: string | null;
|
||||
subscribe: (executionId: string) => void;
|
||||
unsubscribe: () => void;
|
||||
submitDecision: (decision: CoachingDecision, correction?: Record<string, any>, feedback?: string) => void;
|
||||
refreshStats: () => void;
|
||||
}
|
||||
|
||||
const initialStats: CoachingStats = {
|
||||
suggestionsMade: 0,
|
||||
accepted: 0,
|
||||
rejected: 0,
|
||||
corrected: 0,
|
||||
manualExecutions: 0,
|
||||
acceptanceRate: 0,
|
||||
correctionRate: 0,
|
||||
};
|
||||
|
||||
export function useCoachingWebSocket(
|
||||
options: UseCoachingWebSocketOptions = {}
|
||||
): UseCoachingWebSocketReturn {
|
||||
const { serverUrl = 'http://localhost:5000', autoConnect = true } = options;
|
||||
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [isSubscribed, setIsSubscribed] = useState(false);
|
||||
const [currentSuggestion, setCurrentSuggestion] = useState<CoachingSuggestion | null>(null);
|
||||
const [stats, setStats] = useState<CoachingStats>(initialStats);
|
||||
const [lastActionResult, setLastActionResult] = useState<CoachingActionResult | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const socketRef = useRef<Socket | null>(null);
|
||||
const executionIdRef = useRef<string | null>(null);
|
||||
|
||||
// Initialize socket connection
|
||||
useEffect(() => {
|
||||
if (!autoConnect) return;
|
||||
|
||||
const socket = io(serverUrl, {
|
||||
transports: ['websocket', 'polling'],
|
||||
reconnection: true,
|
||||
reconnectionAttempts: 5,
|
||||
reconnectionDelay: 1000,
|
||||
});
|
||||
|
||||
socket.on('connect', () => {
|
||||
console.log('[COACHING WS] Connected');
|
||||
setIsConnected(true);
|
||||
setError(null);
|
||||
});
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
console.log('[COACHING WS] Disconnected');
|
||||
setIsConnected(false);
|
||||
setIsSubscribed(false);
|
||||
});
|
||||
|
||||
socket.on('connect_error', (err) => {
|
||||
console.error('[COACHING WS] Connection error:', err);
|
||||
setError(`Connection error: ${err.message}`);
|
||||
});
|
||||
|
||||
// COACHING specific events
|
||||
socket.on('coaching_subscribed', (data) => {
|
||||
console.log('[COACHING WS] Subscribed:', data);
|
||||
setIsSubscribed(true);
|
||||
if (data.stats) {
|
||||
setStats(convertStats(data.stats));
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('coaching_unsubscribed', () => {
|
||||
console.log('[COACHING WS] Unsubscribed');
|
||||
setIsSubscribed(false);
|
||||
setCurrentSuggestion(null);
|
||||
});
|
||||
|
||||
socket.on('coaching_suggestion', (data: any) => {
|
||||
console.log('[COACHING WS] Suggestion received:', data);
|
||||
setCurrentSuggestion({
|
||||
executionId: data.execution_id,
|
||||
action: data.action,
|
||||
target: data.target || {},
|
||||
params: data.params || {},
|
||||
confidence: data.confidence || 0,
|
||||
alternatives: data.alternatives,
|
||||
screenshotPath: data.screenshot_path,
|
||||
context: data.context,
|
||||
timestamp: data.timestamp,
|
||||
});
|
||||
});
|
||||
|
||||
socket.on('coaching_action_result', (data: any) => {
|
||||
console.log('[COACHING WS] Action result:', data);
|
||||
setLastActionResult({
|
||||
executionId: data.execution_id,
|
||||
action: data.action,
|
||||
success: data.success,
|
||||
result: data.result,
|
||||
error: data.error,
|
||||
timestamp: data.timestamp,
|
||||
});
|
||||
// Clear current suggestion after result
|
||||
setCurrentSuggestion(null);
|
||||
});
|
||||
|
||||
socket.on('coaching_stats_update', (data: any) => {
|
||||
console.log('[COACHING WS] Stats update:', data);
|
||||
if (data.stats) {
|
||||
setStats(convertStats(data.stats));
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('coaching_decision_accepted', (data: any) => {
|
||||
console.log('[COACHING WS] Decision accepted:', data);
|
||||
});
|
||||
|
||||
socket.on('coaching_decision_broadcast', (data: any) => {
|
||||
console.log('[COACHING WS] Decision broadcast:', data);
|
||||
});
|
||||
|
||||
socket.on('coaching_session_end', (data: any) => {
|
||||
console.log('[COACHING WS] Session ended:', data);
|
||||
setIsSubscribed(false);
|
||||
setCurrentSuggestion(null);
|
||||
if (data.stats) {
|
||||
setStats(convertStats(data.stats));
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('error', (data) => {
|
||||
console.error('[COACHING WS] Error:', data);
|
||||
setError(data.message || 'Unknown error');
|
||||
});
|
||||
|
||||
socketRef.current = socket;
|
||||
|
||||
return () => {
|
||||
socket.disconnect();
|
||||
socketRef.current = null;
|
||||
};
|
||||
}, [serverUrl, autoConnect]);
|
||||
|
||||
// Convert backend stats format to frontend format
|
||||
const convertStats = (backendStats: Record<string, any>): CoachingStats => {
|
||||
return {
|
||||
suggestionsMade: backendStats.suggestions_made || 0,
|
||||
accepted: backendStats.accepted || 0,
|
||||
rejected: backendStats.rejected || 0,
|
||||
corrected: backendStats.corrected || 0,
|
||||
manualExecutions: backendStats.manual_executions || 0,
|
||||
acceptanceRate: backendStats.acceptance_rate || 0,
|
||||
correctionRate: backendStats.correction_rate || 0,
|
||||
};
|
||||
};
|
||||
|
||||
// Subscribe to COACHING events for an execution
|
||||
const subscribe = useCallback((executionId: string) => {
|
||||
if (!socketRef.current || !isConnected) {
|
||||
setError('Not connected to server');
|
||||
return;
|
||||
}
|
||||
|
||||
executionIdRef.current = executionId;
|
||||
socketRef.current.emit('subscribe_coaching', { execution_id: executionId });
|
||||
}, [isConnected]);
|
||||
|
||||
// Unsubscribe from COACHING events
|
||||
const unsubscribe = useCallback(() => {
|
||||
if (!socketRef.current || !executionIdRef.current) return;
|
||||
|
||||
socketRef.current.emit('unsubscribe_coaching', {
|
||||
execution_id: executionIdRef.current,
|
||||
});
|
||||
executionIdRef.current = null;
|
||||
}, []);
|
||||
|
||||
// Submit a COACHING decision
|
||||
const submitDecision = useCallback(
|
||||
(decision: CoachingDecision, correction?: Record<string, any>, feedback?: string) => {
|
||||
if (!socketRef.current || !executionIdRef.current) {
|
||||
setError('Not subscribed to any execution');
|
||||
return;
|
||||
}
|
||||
|
||||
socketRef.current.emit('coaching_decision', {
|
||||
execution_id: executionIdRef.current,
|
||||
decision,
|
||||
correction,
|
||||
feedback,
|
||||
});
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
// Refresh stats
|
||||
const refreshStats = useCallback(() => {
|
||||
if (!socketRef.current || !executionIdRef.current) return;
|
||||
|
||||
socketRef.current.emit('get_coaching_stats', {
|
||||
execution_id: executionIdRef.current,
|
||||
});
|
||||
}, []);
|
||||
|
||||
return {
|
||||
isConnected,
|
||||
isSubscribed,
|
||||
currentSuggestion,
|
||||
stats,
|
||||
lastActionResult,
|
||||
error,
|
||||
subscribe,
|
||||
unsubscribe,
|
||||
submitDecision,
|
||||
refreshStats,
|
||||
};
|
||||
}
|
||||
|
||||
export default useCoachingWebSocket;
|
||||
287
visual_workflow_builder/frontend/src/hooks/useCorrectionPacks.ts
Normal file
287
visual_workflow_builder/frontend/src/hooks/useCorrectionPacks.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Hook for Correction Packs API
|
||||
*
|
||||
* Provides methods to interact with the Correction Packs backend API.
|
||||
*/
|
||||
|
||||
import { useState, useCallback, useEffect } from 'react';
|
||||
|
||||
// Types
|
||||
export interface Correction {
|
||||
id: string;
|
||||
actionType: string;
|
||||
elementType: string;
|
||||
failureReason?: string;
|
||||
correctionType: string;
|
||||
originalTarget: Record<string, any>;
|
||||
correctedTarget: Record<string, any>;
|
||||
successCount: number;
|
||||
failureCount: number;
|
||||
confidenceScore: number;
|
||||
createdAt: string;
|
||||
source: {
|
||||
sessionId?: string;
|
||||
workflowId?: string;
|
||||
nodeId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface CorrectionPack {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
corrections: Correction[];
|
||||
tags: string[];
|
||||
category: string;
|
||||
version: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
statistics: {
|
||||
totalCorrections: number;
|
||||
avgConfidence: number;
|
||||
mostCommonType: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface PackStatistics {
|
||||
totalCorrections: number;
|
||||
avgSuccessRate: number;
|
||||
avgConfidence: number;
|
||||
byType: Record<string, number>;
|
||||
byElement: Record<string, number>;
|
||||
}
|
||||
|
||||
interface UseCorrectionPacksReturn {
|
||||
packs: CorrectionPack[];
|
||||
selectedPack: CorrectionPack | null;
|
||||
loading: boolean;
|
||||
error: string | null;
|
||||
fetchPacks: () => Promise<void>;
|
||||
fetchPack: (packId: string) => Promise<CorrectionPack | null>;
|
||||
createPack: (name: string, description?: string, tags?: string[], category?: string) => Promise<CorrectionPack | null>;
|
||||
updatePack: (packId: string, updates: Partial<CorrectionPack>) => Promise<boolean>;
|
||||
deletePack: (packId: string) => Promise<boolean>;
|
||||
exportPack: (packId: string, format?: 'json' | 'yaml') => Promise<void>;
|
||||
importPack: (file: File) => Promise<CorrectionPack | null>;
|
||||
getStatistics: (packId: string) => Promise<PackStatistics | null>;
|
||||
findApplicable: (context: Record<string, any>) => Promise<Correction[]>;
|
||||
selectPack: (pack: CorrectionPack | null) => void;
|
||||
}
|
||||
|
||||
const API_BASE = 'http://localhost:5000/api';
|
||||
|
||||
export function useCorrectionPacks(): UseCorrectionPacksReturn {
|
||||
const [packs, setPacks] = useState<CorrectionPack[]>([]);
|
||||
const [selectedPack, setSelectedPack] = useState<CorrectionPack | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Fetch all packs
|
||||
const fetchPacks = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/correction-packs`);
|
||||
if (!response.ok) throw new Error('Failed to fetch packs');
|
||||
|
||||
const data = await response.json();
|
||||
setPacks(data.packs || []);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Fetch single pack
|
||||
const fetchPack = useCallback(async (packId: string): Promise<CorrectionPack | null> => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/correction-packs/${packId}`);
|
||||
if (!response.ok) return null;
|
||||
|
||||
const data = await response.json();
|
||||
return data.pack || data;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
return null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Create new pack
|
||||
const createPack = useCallback(async (
|
||||
name: string,
|
||||
description?: string,
|
||||
tags?: string[],
|
||||
category?: string
|
||||
): Promise<CorrectionPack | null> => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/correction-packs`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ name, description, tags, category }),
|
||||
});
|
||||
|
||||
if (!response.ok) throw new Error('Failed to create pack');
|
||||
|
||||
const data = await response.json();
|
||||
await fetchPacks(); // Refresh list
|
||||
return data.pack || data;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
return null;
|
||||
}
|
||||
}, [fetchPacks]);
|
||||
|
||||
// Update pack
|
||||
const updatePack = useCallback(async (
|
||||
packId: string,
|
||||
updates: Partial<CorrectionPack>
|
||||
): Promise<boolean> => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/correction-packs/${packId}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(updates),
|
||||
});
|
||||
|
||||
if (!response.ok) return false;
|
||||
|
||||
await fetchPacks(); // Refresh list
|
||||
return true;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
return false;
|
||||
}
|
||||
}, [fetchPacks]);
|
||||
|
||||
// Delete pack
|
||||
const deletePack = useCallback(async (packId: string): Promise<boolean> => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/correction-packs/${packId}`, {
|
||||
method: 'DELETE',
|
||||
});
|
||||
|
||||
if (!response.ok) return false;
|
||||
|
||||
await fetchPacks(); // Refresh list
|
||||
if (selectedPack?.id === packId) {
|
||||
setSelectedPack(null);
|
||||
}
|
||||
return true;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
return false;
|
||||
}
|
||||
}, [fetchPacks, selectedPack]);
|
||||
|
||||
// Export pack
|
||||
const exportPack = useCallback(async (
|
||||
packId: string,
|
||||
format: 'json' | 'yaml' = 'json'
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${API_BASE}/correction-packs/${packId}/export?format=${format}`
|
||||
);
|
||||
|
||||
if (!response.ok) throw new Error('Failed to export pack');
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = `correction_pack_${packId}.${format}`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(a);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Import pack
|
||||
const importPack = useCallback(async (file: File): Promise<CorrectionPack | null> => {
|
||||
try {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
|
||||
const response = await fetch(`${API_BASE}/correction-packs/import`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) throw new Error('Failed to import pack');
|
||||
|
||||
const data = await response.json();
|
||||
await fetchPacks(); // Refresh list
|
||||
return data.pack || data;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
return null;
|
||||
}
|
||||
}, [fetchPacks]);
|
||||
|
||||
// Get statistics
|
||||
const getStatistics = useCallback(async (packId: string): Promise<PackStatistics | null> => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/correction-packs/${packId}/statistics`);
|
||||
if (!response.ok) return null;
|
||||
|
||||
const data = await response.json();
|
||||
return data.statistics || data;
|
||||
} catch (err) {
|
||||
return null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Find applicable corrections
|
||||
const findApplicable = useCallback(async (
|
||||
context: Record<string, any>
|
||||
): Promise<Correction[]> => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/correction-packs/find`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(context),
|
||||
});
|
||||
|
||||
if (!response.ok) return [];
|
||||
|
||||
const data = await response.json();
|
||||
return data.corrections || [];
|
||||
} catch (err) {
|
||||
return [];
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Select pack
|
||||
const selectPack = useCallback((pack: CorrectionPack | null) => {
|
||||
setSelectedPack(pack);
|
||||
}, []);
|
||||
|
||||
// Initial fetch
|
||||
useEffect(() => {
|
||||
fetchPacks();
|
||||
}, [fetchPacks]);
|
||||
|
||||
return {
|
||||
packs,
|
||||
selectedPack,
|
||||
loading,
|
||||
error,
|
||||
fetchPacks,
|
||||
fetchPack,
|
||||
createPack,
|
||||
updatePack,
|
||||
deletePack,
|
||||
exportPack,
|
||||
importPack,
|
||||
getStatistics,
|
||||
findApplicable,
|
||||
selectPack,
|
||||
};
|
||||
}
|
||||
|
||||
export default useCorrectionPacks;
|
||||
Reference in New Issue
Block a user