Merge pull request #279 from transformerlab/add/basic_eval_check

Add Basic Eval Checks

src/renderer/components/Experiment/DynamicPluginForm.tsx

@@ -22,10 +22,11 @@ import {
   Slider,
   Stack,
   Option,
-  Autocomplete
+  Autocomplete,
 } from '@mui/joy';
 import { useMemo } from 'react';
 import ModelProviderWidget from 'renderer/components/Experiment/Widgets/ModelProviderWidget';
+import CustomEvaluationWidget from './Widgets/CustomEvaluationWidget';
 
 import {
   RegistryWidgetsType,
@@ -421,7 +422,13 @@ function CustomAutocompleteWidget<T = any, S extends StrictRJSFSchema = RJSFSche
   // Determine default value.
   const defaultValue = _multiple ? [] : '';
   // Use the provided value or fallback to default.
-  const currentValue = value !== undefined ? value : defaultValue;
+  let currentValue = value !== undefined ? value : defaultValue;
+
+  // Check if currentValue is an array, if a string, convert it to an array.
+  const isString = typeof currentValue === 'string';
+  if (isString) {
+    currentValue = currentValue.split(',');
+  }
 
   // Map enumOptions into objects with label and value.
   const processedOptionsValues = enumOptions.map((opt) =>
@@ -492,6 +499,7 @@ const widgets: RegistryWidgetsType = {
   RangeWidget: CustomRange,
   SelectWidget: CustomSelectSimple,
   AutoCompleteWidget: CustomAutocompleteWidget,
+  EvaluationWidget: CustomEvaluationWidget,
   ModelProviderWidget: ModelProviderWidget
 };
 

src/renderer/components/Experiment/Eval/EvalTasksTable.tsx

@@ -20,7 +20,20 @@ function formatTemplateConfig(script_parameters): ReactElement {
   // Remove the author/full path from the model name for cleanliness
   // const short_model_name = c.model_name.split('/').pop();
   // Set main_task as either or the metric name from the script parameters
-  const main_task = script_parameters.tasks
+  const main_task = (() => {
+    if (script_parameters.tasks) {
+      try {
+        const tasksArray = JSON.parse(script_parameters.tasks);
+        if (Array.isArray(tasksArray)) {
+          return tasksArray.map((task) => task.name).join(', ');
+        }
+      } catch (error) {
+        // Invalid JSON; fall back to the original value
+      }
+      return script_parameters.tasks;
+    }
+    return script_parameters.tasks;
+  })();
   const dataset_name = script_parameters.dataset_name
     ? script_parameters.dataset_name
     : 'N/A';

src/renderer/components/Experiment/Eval/ViewCSVModal.tsx

@@ -75,7 +75,8 @@ function formatEvalData(data) {
 }
 
 function formatArrayOfScores(scores) {
-  const formattedScores = scores.map((score) => {
+  const scoresArray = Array.isArray(scores) ? scores : [scores];
+  const formattedScores = scoresArray.map((score) => {
     const metricName = Object.keys(score)[0];
     const value = Object.values(score)[0];
 

src/renderer/components/Experiment/Widgets/CustomEvaluationWidget.tsx

+import React from 'react';
+import { WidgetProps } from '@rjsf/core';
+import { Button, Input, Select, Option } from '@mui/joy';
+
+type EvaluationField = {
+  name: string;
+  expression: string;
+  return_type: string;
+};
+
+const parseValue = (val: any): EvaluationField[] => {
+  if (Array.isArray(val)) {
+    if (val.every(item => typeof item === "string")) {
+      // If every element is a string: join them and parse the result.
+      try {
+        const joined = val.join(',');
+        const parsed = JSON.parse(joined);
+        return Array.isArray(parsed) ? parsed : [];
+      } catch (err) {
+        console.error("Error parsing evaluation widget value:", err);
+        return [];
+      }
+    } else {
+      // If not all elements are strings, assume it's already an array of EvaluationField.
+      return val;
+    }
+  } else if (typeof val === "string") {
+    try {
+      return JSON.parse(val);
+    } catch (err) {
+      console.error("Error parsing evaluation widget value string:", err);
+      return [];
+    }
+  }
+  return [];
+};
+
+const CustomEvaluationWidget = (props: WidgetProps<any>) => {
+  const { id, value, onChange, disabled, readonly } = props;
+
+  // Directly derive evaluation metrics from the value prop.
+  const evalMetrics: EvaluationField[] = React.useMemo(() => parseValue(value), [value]);
+
+  const handleAddField = () => {
+    const updatedMetrics = [
+      ...evalMetrics,
+      { name: '', expression: '', return_type: 'boolean' }
+    ];
+    onChange(updatedMetrics);
+  };
+
+  const handleFieldChange = (
+    index: number,
+    field: keyof EvaluationField,
+    newValue: string
+  ) => {
+    const updated = evalMetrics.map((evaluation, i) =>
+      i === index ? { ...evaluation, [field]: newValue } : evaluation
+    );
+    onChange(updated);
+  };
+
+  const handleRemoveField = (index: number) => {
+    const updated = evalMetrics.filter((_, i) => i !== index);
+    onChange(updated);
+  };
+
+  return (
+    <div id={id}>
+      {evalMetrics.map((evaluation, index) => (
+        <div
+          key={index}
+          style={{
+            marginBottom: '1rem',
+            border: '1px solid #ccc',
+            padding: '0.5rem'
+          }}
+        >
+          <Input
+            placeholder="Evaluation Name"
+            value={evaluation.name}
+            onChange={(e) =>
+              handleFieldChange(index, 'name', e.target.value)
+            }
+            disabled={disabled || readonly}
+            style={{ marginBottom: '0.5rem' }}
+          />
+          <textarea
+            placeholder="Regular Expression/String"
+            value={evaluation.expression}
+            onChange={(e) =>
+              handleFieldChange(index, 'expression', e.target.value)
+            }
+            disabled={disabled || readonly}
+            style={{ marginBottom: '0.5rem' }}
+          />
+          <Select
+            placeholder="Output Type"
+            value={evaluation.return_type}
+            onChange={(e, newValue) =>
+              handleFieldChange(index, 'return_type', newValue as string)
+            }
+            disabled={disabled || readonly}
+            style={{ marginBottom: '0.5rem' }}
+          >
+            <Option value="boolean">Boolean</Option>
+            <Option value="number">Number</Option>
+            <Option value="contains">Contains</Option>
+            <Option value="isequal">IsEqual</Option>
+          </Select>
+          <Button
+            onClick={() => handleRemoveField(index)}
+            disabled={disabled || readonly}
+            size="sm"
+            variant="outlined"
+          >
+            Remove Field
+          </Button>
+        </div>
+      ))}
+      <Button
+        onClick={handleAddField}
+        disabled={disabled || readonly}
+        variant="solid"
+      >
+        Add Field
+      </Button>
+      {/* Hidden input to capture the JSON result on form submission */}
+      <input type="hidden" id={id} name={id} value={JSON.stringify(evalMetrics)} />
+    </div>
+  );
+};
+
+export default CustomEvaluationWidget;