CyberX8/CyberX8_RT/Modules/Rinse/RinseEntity.cs

using Aitex.Core.RT.DataCenter;
using Aitex.Core.RT.Device;
using Aitex.Core.RT.Fsm;
using Aitex.Core.RT.Log;
using Aitex.Core.RT.OperationCenter;
using Aitex.Core.Util;
using Aitex.Core.Utilities;
using MECF.Framework.Common.Equipment;
using MECF.Framework.Common.SubstrateTrackings;
using MECF.Framework.Common.WaferHolder;
using CyberX8_Core;
using CyberX8_RT.Devices.Rinse;
using System;
using System.Collections.Generic;
using System.Linq;
using MECF.Framework.Common.Persistent.Rinse;
using MECF.Framework.Common.RecipeCenter;
using Aitex.Core.RT.RecipeCenter;
using CyberX8_RT.Devices.Safety;
using Aitex.Core.RT.SCCore;
using MECF.Framework.Common.Alarm;
using MECF.Framework.Common.CommonData;
using CyberX8_RT.Schedulers;
using CyberX8_RT.Modules.Dryer;
using MECF.Framework.Common.ProcessCell;

namespace CyberX8_RT.Modules.Rinse
{
    public class RinseEntity : Entity, IEntity, IModuleEntity
    {
        #region 常量 
        private const string AUTO = "Auto";
        private const string MANUAL = "Manual";
        private const string DISABLED = "Disabled";
        private const string ENGINEERING = "Engineering";
        private const string PRODUCTION = "Production";
        #endregion

        #region 内部变量
        /// <summary>
        /// Rinse设备
        /// </summary>
        private RinseDevice _rinseDevice;
        /// <summary>
        /// Cycle次数
        /// </summary>
        private int _cycle = 0;
        /// <summary>
        /// 已经完成的Cycle次数
        /// </summary>
        private int _achievedCycle = 0;
        /// <summary>
        /// Manual模式下Cycle run 
        /// </summary>
        private RinseCycleManualProcessRecipeRoutine _cycleManualProcessRoutine;
        /// <summary>
        /// Keepwet Routine
        /// </summary>
        private RinseKeepwetRoutine _keepwetRoutine;
        /// <summary>
        /// Recipe时长
        /// </summary>
        private Dictionary<string, int> _dictRecipeTime = new Dictionary<string, int>();
        /// <summary>
        /// 持久性对象
        /// </summary>
        private RinsePersistentValue _persistentValue;
        /// <summary>
        /// 当前Recipe
        /// </summary>
        private QdrRecipe _currentRecipe;
        /// <summary>
        /// Recipe时长
        /// </summary>
        private int _recipeTime;
        /// <summary>
        /// Runrecipe已经运行的时间
        /// </summary>
        private int _runrecipeElapsedTime = 0;
        /// <summary>
        /// run recipe start time
        /// </summary>
        private DateTime _runRecipeStartTime;
        /// <summary>
        ///  run recipe complete time
        /// </summary>
        private DateTime _runRecipeCompleteTime;
        /// <summary>
        /// 是否Retry
        /// </summary>
        private bool _isRetry = false;
        /// <summary>
        /// 当前recipe是否完成
        /// </summary>
        private bool _isCurrentReceipeComplete;
        /// <summary>
        ///  当前wafershuttle开始浸泡时间
        /// </summary>
        private DateTime _currentWaferShuttleStartSoakTime;
        /// <summary>
        /// Wafershuttle最浸泡超时waring时长
        /// </summary>
        private int _waferShuttleSoakMaxTime;

        #region 计算recipe运行时间相关的配置文件参数
        /// <summary>
        /// Concurrent Fill Time Seconds
        /// </summary>
        private int _concurrentFillTimeSeconds;
        /// <summary>
        /// 开始注水的偏差
        /// </summary>
        private double _fillingStartedDelta;
        /// <summary>
        /// 注满数值
        /// </summary>
        private int _sensorReadingFull;
        /// <summary>
        /// 液位为空数值
        /// </summary>
        private int _sensorReadingEmpty;
        /// <summary>
        /// 从开始排水到检测是否排空的间隔时间
        /// </summary>
        private int _normalDrainTimeSeconds;
        /// <summary>
        /// 开始注水后检测是否正常注水间隔
        /// </summary>
        private int _checkIsFillingTimeSeconds;
        /// <summary>
        /// 开始注水后检测是否注满间隔
        /// </summary>
        private int _checkIsFullTimeSeconds;
        /// <summary>
        /// Clamp Cycle开关次数
        /// </summary>
        private int _numberClampCyclesToComplete;
        /// <summary>
        /// Clamp开关等待时间
        /// </summary>
        private double _clampCycleTimeSeconds;
        #endregion

        #endregion

        #region 属性
        /// <summary>
        /// 模块名称
        /// </summary>
        public ModuleName Module { get; private set; }
        /// <summary>
        /// 初始化状态
        /// </summary>
        public bool IsInit
        {
            get { return fsm.State == (int)RinseState.Init; }
        }
        /// <summary>
        /// 是否完成初始化
        /// </summary>
        public bool IsInitialized
        {
            get { return fsm.State >= (int)RinseState.Initialized; }
        }
        /// <summary>
        /// 空闲状态
        /// </summary>
        public bool IsIdle
        {
            get
            {
                return fsm.State == (int)RinseState.Idle;
            }
        }
        /// <summary>
        /// 是否发生错误
        /// </summary>
        public bool IsError
        {
            get { return fsm.State == (int)RinseState.Error; }
        }
        /// <summary>
        /// 是否正在作业
        /// </summary>
        public bool IsBusy { get { return fsm.State > (int)RinseState.Idle; } }


        /// <summary>
        /// WaferHolder信息
        /// </summary>
        public WaferHolderInfo WaferHolderInfo { get { return WaferHolderManager.Instance.GetWaferHolder(Module.ToString()); } }
        /// <summary>
        /// 已完成的RunRecipeCycle次数
        /// </summary>
        public int AchievedCycle { get { return _achievedCycle; } }
        /// <summary>
        /// 当前状态机状态
        /// </summary>
        public int State { get { return fsm.State; } }
        /// <summary>
        /// Recipe时长
        /// </summary>
        public int RecipeTime
        {
            get { return _recipeTime; } 
        }
        /// <summary>
        /// 剩余时间
        /// </summary>
        public override int TimeToReady
        {
            get
            {
                if (_currentRecipe == null)
                {
                    return base.TimeToReady;
                }
                switch (fsm.State)
                {
                    case (int)RinseState.RunReciping:
                        return Math.Max(_recipeTime - fsm.ElapsedTime / 1000, 0);
                    default:
                        return base.TimeToReady;
                }
            }
        }
        /// <summary>
        /// 是否禁用
        /// </summary>
        public bool IsDisable { get { return _persistentValue == null || _persistentValue.OperatingMode == DISABLED; } }

        /// <summary>
        /// 自动模式
        /// </summary>
        public bool IsAuto { get { return _persistentValue != null && _persistentValue.OperatingMode == AUTO; } }
        /// <summary>
        /// 自动模式
        /// </summary>
        public bool IsManual { get { return _persistentValue != null && _persistentValue.OperatingMode == MANUAL; } }
        /// <summary>
        /// 是否为工程模式
        /// </summary>
        public bool IsEngineering { get { return _persistentValue != null && _persistentValue.RecipeOperatingMode == ENGINEERING; } }
        /// <summary>
        /// 是否为产品模式
        /// </summary>
        public bool IsProduction { get { return _persistentValue != null && _persistentValue.RecipeOperatingMode == PRODUCTION; } }
        #endregion

        /// <summary>
        /// 构造函数
        /// </summary>
        /// <param name="module"></param>
        public RinseEntity(ModuleName module)
        {
            Module = module;
            _rinseDevice=DEVICE.GetDevice<RinseDevice>(Module.ToString());
            WaferManager.Instance.SubscribeLocation(Module, 2);
            //相关配置文件
           

            InitialFsm();

        }
        /// <summary>
        /// 初始化
        /// </summary>
        /// <returns></returns>
        protected override bool Init()
        {
            InitializeParameter();
            InitializeDATA();
            InitializeRoutine();
            InitializeOperation();
            InitializeRoutine();
            return true;
        }

        /// <summary>
        /// 初始化状态机
        /// </summary>
        private void InitialFsm()
        {
            fsm = new StateMachine<RinseEntity>(Module.ToString(), (int)RinseState.Init, 20);
            fsm.EnableRepeatedMsg(true);

            AnyStateTransition(RinseMsg.Error, ErrorSolution, RinseState.Error);

            Transition(RinseState.Error, RinseMsg.ResumeError, (param) => { return true; }, RinseState.Init);

            //Initialized
            AnyStateTransition(RinseMsg.Initialize, InitializeAll, RinseState.Initialized);
            //直接进入Idle
            Transition(RinseState.Initialized, FSM_MSG.TIMER, NullFunc, RinseState.Idle);
            //CycleRunRecipe
            Transition(RinseState.Idle, RinseMsg.CycleProcessRecipe, RunRecipeProcess, RinseState.CycleManualProcessing);
            Transition(RinseState.CycleManualProcessing, FSM_MSG.TIMER, RunRecipeMonitor, RinseState.Idle);
            Transition(RinseState.CycleManualProcessing,RinseMsg.Abort,RunRecipeAbort, RinseState.Idle);
            //RunRecipe
            Transition(RinseState.Idle, RinseMsg.RunRecipe, RunRecipeProcess, RinseState.RunReciping);
            Transition(RinseState.RunReciping, FSM_MSG.TIMER, RunRecipeMonitor, RinseState.RunRecipeComplete);
            Transition(RinseState.RunRecipeComplete, RinseMsg.RecipeComplete, NullFunc, RinseState.Idle);
            Transition(RinseState.RunReciping, RinseMsg.Abort, RunRecipeAbort, RinseState.Abort);
            Transition(RinseState.Idle, FSM_MSG.TIMER, RinseCompleteSoakTimeMonitor, RinseState.Idle);
            //KeepWet
            Transition(RinseState.RunRecipeComplete, RinseMsg.Keepwet, Keepwet, RinseState.KeepWeting);
            Transition(RinseState.KeepWetComplete, RinseMsg.Keepwet, Keepwet, RinseState.KeepWeting);
            Transition(RinseState.KeepWeting, FSM_MSG.TIMER, KeepwetMonitor, RinseState.KeepWetComplete);
            Transition(RinseState.KeepWeting, RinseMsg.RecipeComplete, KeepwetingComplete, RinseState.Idle);
            //Enter Init
            Transition(RinseState.Idle, RinseMsg.Init, NullFunc, RinseState.Init);

            //Retry
            Transition(RinseState.Error, RinseMsg.Retry, NullFunc, RinseState.Retrying);
            Transition(RinseState.Retrying, FSM_MSG.TIMER, RinseRetry, RinseState.Retrying);
            Transition(RinseState.Retrying, RinseMsg.RunRecipe, RetryRunRecipe, RinseState.RunReciping);

            EnumLoop<RinseState>.ForEach((item) => { fsm.MapState((int)item, item.ToString()); });
            EnumLoop<RinseMsg>.ForEach((item) => { fsm.MapMessage((int)item, item.ToString()); });
        }

        /// <summary>
        /// 初始化参数
        /// </summary>
        private void InitializeParameter()
        {
            _persistentValue = RinsePersistentManager.Instance.GetRinsePersistentValue(Module.ToString());
            if (_persistentValue == null)
            {
                LOG.WriteLog(eEvent.ERR_RESERVOIR, Module.ToString(), "Persistent Value Object is not exist");
            }
            _isCurrentReceipeComplete = false;
            _waferShuttleSoakMaxTime = SC.GetValue<int>($"QDR.PickupDelayWarningSeconds");
        }
        /// <summary>
        /// 初始化DATA
        /// </summary>
        private void InitializeDATA()
        {
            InitializeSvid();
            DATA.Subscribe($"{Module}.FsmState", () => ((RinseState)fsm.State).ToString(), SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.WaferHolder", () => WaferHolderInfo, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.IsInit", () => IsInit, SubscriptionAttribute.FLAG.IgnoreSaveDB); 
            DATA.Subscribe($"{Module}.IsIdle", () => IsIdle, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.IsError", () => IsError, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.IsBusy", () => IsBusy, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.IsDisable", () => IsDisable, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.AchievedCycle", () => AchievedCycle, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.CurrentRecipe", () => _currentRecipe != null ? _currentRecipe.Ppid : "", SubscriptionAttribute.FLAG.IgnoreSaveDB);
        }
        /// <summary>
        /// 初始化SVID
        /// </summary>
        private void InitializeSvid()
        {
            DATA.Subscribe($"{Module}.State", () => ((RinseState)fsm.State).ToString(), SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.LotID", () => WaferHolderInfo?.LotId, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.WSID", () => WaferHolderInfo?.Id, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.LSAID", () => WaferHolderInfo?.CrsAId, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.LSBID", () => WaferHolderInfo?.CrsBId, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.ModuleRecipe", () =>  _currentRecipe?.Ppid, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.SequenceRecipe", () => WaferHolderInfo?.SequenceId, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.WaferAID", () => WaferHolderInfo?.WaferAId, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.WaferBID", () => WaferHolderInfo?.WaferBId, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.TotalTime", () => _recipeTime, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.TimeRemain", () => _recipeTime != 0 ? (_recipeTime - Math.Round((double)_cycleManualProcessRoutine.ElapsedMilliseconds / 1000, 0)) : 0, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.Task", () => WaferHolderInfo?.CurrentControlJobId, SubscriptionAttribute.FLAG.IgnoreSaveDB);
            DATA.Subscribe($"{Module}.OperatingMode", () => _persistentValue != null ? _persistentValue.OperatingMode : "None", SubscriptionAttribute.FLAG.IgnoreSaveDB); 
        }
        /// <summary>
        /// 初始化Routine
        /// </summary>
        private void InitializeRoutine()
        {
            _cycleManualProcessRoutine = new RinseCycleManualProcessRecipeRoutine(Module.ToString());
            _keepwetRoutine = new RinseKeepwetRoutine(Module.ToString());
        }
        /// <summary>
        /// 初始化操作
        /// </summary>
        private void InitializeOperation()
        {
            OP.Subscribe($"{Module}.InitializeAll", (cmd, args) => { return CheckToPostMessage<RinseState, RinseMsg>(eEvent.ERR_RINSE, Module.ToString(), (int)RinseMsg.Initialize); });
            OP.Subscribe($"{Module}.CycleManualProcessRecipe", (cmd, args) => 
            {
                QdrRecipe recipe = RecipeFileManager.Instance.LoadGenericityRecipe<QdrRecipe>(args[0].ToString());
                if (recipe == null)
                {
                    LOG.WriteLog(eEvent.ERR_RINSE, Module.ToString(), $"{args[0]} recipe is null");
                    return false;
                }
                object[] objects = new object[args.Length];
                objects[0] = recipe;
                for (int i = 1; i < args.Length; i++)
                {
                    objects[i] = args[i];
                }
                return CheckToPostMessage<RinseState, RinseMsg>(eEvent.ERR_RINSE, Module.ToString(), (int)RinseMsg.CycleProcessRecipe, objects);
            });
            OP.Subscribe($"{Module}.Abort", (cmd, args) => { return CheckToPostMessage<RinseState, RinseMsg>(Aitex.Core.RT.Log.eEvent.ERR_RINSE, Module.ToString(), (int)RinseMsg.Abort); });
            
        }
        /// <summary>
        /// EnterInit
        /// </summary>
        public void EnterInit()
        {
            if ((RinseState)fsm.State != RinseState.Idle) return;
            else
            {
                CheckToPostMessage<RinseState, RinseMsg>(eEvent.ERR_RINSE, Module.ToString(), (int)RinseMsg.Init);
            }
        }
        #region Initialized
        /// <summary>
        /// Initialize
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool InitializeAll(object[] param)
        {
            if (fsm.State == (int)RinseState.Initializing)
            {
                LOG.WriteLog(eEvent.WARN_RINSE, Module.ToString(), "state is Initializing,cannot do initialize");
                return false;
            }
            _rinseDevice.UpdateStateMachine("Idle");
            _rinseDevice.UpdateStatus("Idle");
            if (!_rinseDevice.IOInitialized)
            {
                return false;
            }
            return _rinseDevice.IOInitialized;
        }


        #endregion

        private bool ErrorSolution(object[] param)
        {
            bool result = _rinseDevice.FillValveOff();
            if (!result)
            {
                LOG.WriteLog(eEvent.ERR_RINSE, "Close Fill Valve error");
            
            }
            result = _rinseDevice.N2ValveOff();
            if (!result)
            {
                LOG.WriteLog(eEvent.ERR_RINSE, "Close N2 Valve error");
            }
            return true;
        }

        #region RunRecipe
        /// <summary>
        /// Run Recipe
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool RunRecipeProcess(object[] param)
        {
            _concurrentFillTimeSeconds = SC.GetValue<int>("QDR.ConcurrentFillTimeSeconds");
            _fillingStartedDelta = SC.GetValue<double>("QDR.FillingStartedDelta");
            _sensorReadingFull = SC.GetValue<int>("QDR.SensorReadingFull");
            _sensorReadingEmpty = SC.GetValue<int>("QDR.SensorReadingEmpty");
            _normalDrainTimeSeconds = SC.GetValue<int>("QDR.NominalDrainTimeSeconds");
            _checkIsFillingTimeSeconds = SC.GetValue<int>("QDR.NominalCheckFillWaterTimeSeconds");
            _checkIsFullTimeSeconds = SC.GetValue<int>("QDR.NominalCheckFillFullTimeSeconds");
            _numberClampCyclesToComplete = SC.GetValue<int>("QDR.NumberClampCyclesToComplete");
            _clampCycleTimeSeconds = SC.GetValue<double>("QDR.ClampCycleTimeSeconds");
            
            _cycle = (int)param[1];
            QdrRecipe recipe = param[0] as QdrRecipe;
            bool result = _cycleManualProcessRoutine.Start(param) == RState.Running;
            _runrecipeElapsedTime = 0;
            if (result)
            {
                //_recipeTime = 0;
                _isRetry = false;
                _recipeTime = CalculateRunRecipeTime(recipe, _cycle);
                _currentRecipe = recipe;
                _runRecipeStartTime = DateTime.Now;
                if (WaferHolderInfo != null && _currentRecipe != null)
                {
                    FaModuleNotifier.Instance.NotifyWaferShuttleRecipeStart(WaferHolderInfo, _currentRecipe.Ppid);
                }
            }
            return result;
        }

        private int CalculateRunRecipeTime(QdrRecipe _recipe, int cycle)
        {
            int totaltime = 0;
            Dictionary<string, int> processTimeDic = new Dictionary<string, int>();
            processTimeDic.Add("RinseFirstStep.WaitDrianEmpty", _recipe.DumpTimeSeconds);
            if (_recipe.Step1N2BubbleOn)
            {
                processTimeDic.Add("RinseFirstStep.N2BubbleOnDelay", _recipe.N2ChargeTimeSeconds);
            }
            processTimeDic.Add("RinseFirstStep.Recipe_FirstRinseCycleWaitEndFillTime", _concurrentFillTimeSeconds * _recipe.Step1NumberOfRinse);
            processTimeDic.Add("RinseFirstStep.Recipe_FirstRinseCycleWaitDwell", _recipe.Step1DwellTimeSeconds * _recipe.Step1NumberOfRinse);
            processTimeDic.Add("RinseFirstStep.Recipe_FirstRinseCycleWaitDrainExtraTime", _recipe.DumpTimeSeconds * _recipe.Step1NumberOfRinse);

            if (!_recipe.FinalRinseDry) //没有第三步rinse
            {
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleWaitEndFillTime", _concurrentFillTimeSeconds * _recipe.Step2NumberOfRinse);
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleWaitDwell", _recipe.Step2DwellTimeSeconds * (_recipe.Step2NumberOfRinse - 1));
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleWaitDrainExtraTime", _recipe.DumpTimeSeconds * _recipe.Step2NumberOfRinse);
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleWaitResistivityStart", _recipe.ResistivityStartTimeSeconds);
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleResistivityAveraging", _recipe.ResistivityDurationSeconds);
            }
            else  //有第三步rinse
            {
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleWaitEndFillTime", _concurrentFillTimeSeconds * _recipe.Step2NumberOfRinse);
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleWaitDwell", _recipe.Step2DwellTimeSeconds * _recipe.Step2NumberOfRinse);
                processTimeDic.Add("RinseSecondStep.Recipe_SecondRinseCycleWaitDrainExtraTime", _recipe.DumpTimeSeconds * _recipe.Step2NumberOfRinse);

                processTimeDic.Add("RinseThirdStep.Recipe_ThirdRinseWaitEndFillTime", _concurrentFillTimeSeconds);
                processTimeDic.Add("RinseThirdStep.Recipe_ThirdRinseCycleWaitSlowDrainTime", _recipe.FinalRinseSlowDrainTime);
                processTimeDic.Add("RinseThirdStep.Recipe_ThirdRinseCycleWaitDrainExtraTime", _recipe.DumpTimeSeconds);
            }
            totaltime = processTimeDic.Sum(x => x.Value);
            return totaltime * cycle;
        }

        /// <summary>
        /// Retry RunRecipe
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool RetryRunRecipe(object[] param)
        {
            int stepIndex = (int)param[0];
            bool result = _cycleManualProcessRoutine.Retry(stepIndex) == RState.Running;
            if (result)
            {
                _isRetry = true;
                if (_currentRecipe != null)
                {
                    _recipeTime = CalculateRunRecipeTime(_currentRecipe, _cycle);
                }
            }
            return result;
        }
        /// <summary>
        /// 监控
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool RunRecipeMonitor(object[] param)
        {
            RState state = _cycleManualProcessRoutine.Monitor();

            if (Singleton<RouteManager>.Instance.IsAutoRunning&&_runrecipeElapsedTime!=TimeToReady)
            {
                _runrecipeElapsedTime =TimeToReady;
                LOG.WriteLog(eEvent.INFO_RINSE, Module.ToString(), $"{WaferHolderInfo?.Id} {Module} RunRecipe TimeToReady {_recipeTime-fsm.ElapsedTime/1000}s.");
            }
            if (state == RState.Failed || state == RState.Timeout)
            {
                if (Singleton<RouteManager>.Instance.IsAutoRunning)
                {
                    //不存在其他可用的Rinse
                    if (!CheckOtherRinseAvaible())
                    {
                        AlarmList alarmList = new AlarmList(Module.ToString(), ((RinseState)fsm.State).ToString(), (int)RinseMsg.RunRecipe,
                        _cycleManualProcessRoutine.ErrorMsg, _cycleManualProcessRoutine.ErrorStep, (int)AlarmType.Error);
                        AlarmListManager.Instance.AddAlarm(alarmList);
                    }
                }
                PostMsg(RinseMsg.Error);
                _rinseDevice.UpdateStateMachine("Error");
                _rinseDevice.UpdateStatus("Error");
                _runrecipeElapsedTime = 0;

                //记录LotTrack
                _runRecipeCompleteTime = DateTime.Now;
                _cycleManualProcessRoutine.RinseLotTrackHeaderDatas.ProcessTime = (_runRecipeCompleteTime - _runRecipeStartTime).TotalSeconds.ToString("F2");
                RinseLotTrackUtil.ExportRinseLotTrack(Module.ToString(), _cycleManualProcessRoutine.RinseLotTrackDatas,
                    _cycleManualProcessRoutine.RinseLotTrackHeaderDatas, IsAuto, _isRetry);
                if (WaferHolderInfo != null && _currentRecipe != null)
                {
                    FaModuleNotifier.Instance.NotifyWaferShuttleRecipeFailed(WaferHolderInfo, _currentRecipe.Ppid);
                }
                return false;
            }
            _achievedCycle = _cycleManualProcessRoutine.GetAchievedCycle();
            bool result = state == RState.End;
            if (result)
            {
                _runrecipeElapsedTime = 0;
                if (Singleton<RouteManager>.Instance.IsAutoRunning)
                {
                    AlarmListManager.Instance.CheckModuleAlamAndRemove(Module.ToString(), RinseState.RunReciping.ToString());
                }
                //记录LotTrack
                _runRecipeCompleteTime = DateTime.Now;
                int timeLength = (int)(_runRecipeCompleteTime - _runRecipeStartTime).TotalSeconds;
                _cycleManualProcessRoutine.RinseLotTrackHeaderDatas.ProcessTime = (_runRecipeCompleteTime - _runRecipeStartTime).TotalSeconds.ToString("F2");
                RinseLotTrackUtil.ExportRinseLotTrack(Module.ToString(), _cycleManualProcessRoutine.RinseLotTrackDatas,
                    _cycleManualProcessRoutine.RinseLotTrackHeaderDatas, IsAuto, _isRetry);
                if (WaferHolderInfo != null && _currentRecipe != null)
                {
                    FaModuleNotifier.Instance.NotifyWaferShuttleRecipeEnd(WaferHolderInfo, _currentRecipe.Ppid, timeLength);
                    if (_persistentValue.OperatingMode == "Auto")
                    {
                        _isCurrentReceipeComplete = true;
                        _currentWaferShuttleStartSoakTime = DateTime.Now;
                    }
                }
            }
            return result;
        }
        /// <summary>
        /// 校验其他rinse是否可用
        /// </summary>
        /// <returns></returns>
        private bool CheckOtherRinseAvaible()
        {
            ModuleName metalName = WaferHolderInfo.MetalModuleName;
            if (metalName!=ModuleName.Unknown)
            {
                return SchedulerSequenceManager.Instance.GetAvaibleModuleCell(_persistentValue.RecipeOperatingMode,ModuleType.Rinse, metalName)!=ModuleName.Unknown;
            }
            else
            {
                return false;
            }
        }
        private bool RunRecipeAbort(object[] param)
        {
            _cycleManualProcessRoutine.Abort();
            _rinseDevice.UpdateStateMachine("Abort");
            _rinseDevice.UpdateStatus("Abort");
            //记录LotTrack
            _runRecipeCompleteTime = DateTime.Now;
            _cycleManualProcessRoutine.RinseLotTrackHeaderDatas.ProcessTime = (_runRecipeCompleteTime - _runRecipeStartTime).TotalSeconds.ToString("F2");
            RinseLotTrackUtil.ExportRinseLotTrack(Module.ToString(), _cycleManualProcessRoutine.RinseLotTrackDatas,
                _cycleManualProcessRoutine.RinseLotTrackHeaderDatas, IsAuto, _isRetry);
            return true;
        }
        /// <summary>
        /// Rinse完成recipe后监控wafershuttle浸泡时间
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool RinseCompleteSoakTimeMonitor(object[] param)
        {
            if (WaferHolderInfo != null && _persistentValue.OperatingMode == "Auto")
            {
                double sockSeconds = (DateTime.Now - _currentWaferShuttleStartSoakTime).TotalSeconds;
                if (_isCurrentReceipeComplete && sockSeconds > _waferShuttleSoakMaxTime)
                {
                    if (!AlarmListManager.Instance.IsContainDataWarn(Module.ToString(), "RinseSockTime"))
                    {
                        AlarmListManager.Instance.AddWarn(Module.ToString(),
                           $"RinseSockTime", $"Current waferShuttle:{WaferHolderInfo.Id} sock time is over PickupDelayWarningSeconds:{_waferShuttleSoakMaxTime} seconds");
                        LOG.WriteLog(eEvent.WARN_RINSE, Module.ToString(), $"Current waferShuttle:{WaferHolderInfo.Id} sock time is over PickupDelayWarningSeconds:{_waferShuttleSoakMaxTime} seconds");
                    }
                }
            }
            else
            {
                _isCurrentReceipeComplete = false;  //wafershuttle拿走后重置当前recipe是否完成变量
            }
            return true;
        }
        #endregion

        #region Keepwet
        /// <summary>
        /// keepwet
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool Keepwet(object[] param)
        {
            return _keepwetRoutine.Start(param) == RState.Running;
        }
        /// <summary>
        /// 停止Keepwet
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool KeepwetingComplete(object[] param)
        {
        //    _keepwetRoutine.Abort();
            return true;
        }
        /// <summary>
        /// keepwet
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool KeepwetMonitor(object[] param)
        {
            RState ret = _keepwetRoutine.Monitor();
            if (ret == RState.End)
            {
                //导出lotTrack数据
                LotTrackFileHeaderCommonData headerCommonData = new LotTrackFileHeaderCommonData();
                if (SC.ContainsItem("System.ToolID")) headerCommonData.ToolID = SC.GetStringValue("System.ToolID");
                headerCommonData.SoftWareVersion = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString();
                RinseLotTrackUtil.ExportRinseLotTrack(Module.ToString(), _keepwetRoutine.RinseLotTrackDatas,
                    headerCommonData, IsAuto, false);
                return true;
            }
            if (ret == RState.Failed || ret == RState.Timeout)
            {
                //导出lotTrack数据
                LotTrackFileHeaderCommonData headerCommonData = new LotTrackFileHeaderCommonData();
                if (SC.ContainsItem("System.ToolID")) headerCommonData.ToolID = SC.GetStringValue("System.ToolID");
                headerCommonData.SoftWareVersion = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString();
                RinseLotTrackUtil.ExportRinseLotTrack(Module.ToString(), _keepwetRoutine.RinseLotTrackDatas,
                    headerCommonData, IsAuto, false);
                PostMsg(RinseMsg.Error);
            }
            return false;
        }
        #endregion

        #region RinseRetry
        /// <summary>
        /// Retry
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private bool RinseRetry(object[] param)
        {
            AlarmList alarmList = AlarmListManager.Instance.GetAlarmListByModule(Module.ToString());
            if (alarmList != null)
            {
                CheckToPostMessage<RinseState, RinseMsg>(eEvent.WARN_RINSE, Module.ToString(), alarmList.ModuleCmd,
                    alarmList.ModuleStep);
            }
            return false;
        }
        #endregion

        public bool Check(int msg, out string reason, params object[] args)
        {
            reason = "";
            return true;
        }

        public bool CheckAcked(int msg)
        {
            return true;
        }

        public int Invoke(string function, params object[] args)
        {
            switch (function)
            {
                case "HomeAll":
                    if (IsIdle)
                    {
                        return (int)FSM_MSG.NONE;
                    }
                    if (CheckToPostMessage<RinseState, RinseMsg>(eEvent.ERR_RINSE, Module.ToString(), (int)RinseMsg.Initialize))
                    {
                        return (int)RinseMsg.Initialize;
                    }
                    else
                    {
                        return (int)FSM_MSG.NONE;
                    }

                case "Retry":
                    if (CheckToPostMessage<RinseState, RinseMsg>(eEvent.ERR_RINSE, Module.ToString(), (int)RinseMsg.Retry))
                    {
                        return (int)RinseMsg.Retry;
                    }
                    else
                    {
                        return (int)FSM_MSG.NONE;
                    }
            }
            return (int)FSM_MSG.NONE;
        }
    }

    public enum RinseMsg
    {
        Error,
        Initialize,
        ResumeError,
        CycleProcessRecipe,
        RunRecipe,
        Abort,
        Keepwet,
        RecipeComplete,
        Init,
        Retry
    }
}