PunkHPX8/Framework/Common/Routine/RoutineRunner.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Diagnostics;
using Aitex.Core.RT.Log;
using MECF.Framework.Common.Equipment;
using Aitex.Core.RT.Routine;
using System.Threading;
using PunkHPX8_Core;

namespace MECF.Framework.Common.Routine
{
    public class RoutineRunner 
    {
        public RState       Status                              => _runnerState;
        private bool        IsSubStepRunning(Enum id)           => _runnerState == RState.Running && _subState == RState.Running && (_curStep != null && _curStep.GetHashCode() == id.GetHashCode());
        private bool        bAllowSubStepStart(Enum id)         => _runnerState == RState.Running && _subState == RState.End && !_steps.Contains(id) && _isLoopMode == false;
        private bool        bAllowLoopSubStepStart(Enum id)     => _runnerState == RState.Running && _subState == RState.End && !_steps.Contains(id) && _isLoopMode;

        public int LoopCounter => _loopCounter;
        public long StepElapsedMS => _subStepTimer.ElapsedMilliseconds;

        private RState      _subState       = RState.Init;
        private Stack<Enum>  _steps         = new Stack<Enum>();
        private Stack<Enum>  _loopSteps     = new Stack<Enum>();

        enum InvalidStep { Invalid = 0x7FFFFFFF, };
        public Enum CurrentStep => _curStep;
        private Enum        _curStep        = InvalidStep.Invalid;
        private RState      _runnerState    = RState.Init;
        private string      _name;
        private string      _loopName;
        private string  _module;

        // 缺省最大超时 10 分钟, delay 5 秒
        private const int   _defaultTimeout = 600000;
        private const int   _defaultDelay = 200;
        private const int _defaultDelay_5s = 5000;

        //用于计时
        Stopwatch           _subStepTimer   = new Stopwatch();

        private bool        _isLoopMode     = false;
        private int         _loopCounterSP  = 0;
        private int         _loopCounter    = 0;
        private int _subRoutineIndex = -1;
        private IRoutine _subRoutine = null;
        static private int  _RunnerToken    = 0;

        private Stopwatch _stopwatch = new Stopwatch();

        private string _errorMsg = "";
        public string ErrorMsg { get{ return _errorMsg; } set { _errorMsg = value; } }
        /// <summary>
        /// 完成时长
        /// </summary>
        public long ElapsedMS =>_stopwatch.ElapsedMilliseconds;

        public RoutineRunner()
        {
            _runnerState = RState.Init;
        }

        public void Reset()
        {
            _runnerState = RState.Init;
            _subState = RState.Init;
            _isLoopMode = false;
            _steps.Clear();
            _loopSteps.Clear();
            _stopwatch.Reset();
            _subStepTimer.Reset();
            _errorMsg = "";
        }

        public RState Start(ModuleName module, string name)
        {
            Reset();
            _module = module.ToString();
            _name = $"{name} (Token:{_RunnerToken++})";
            _runnerState = RState.Running;
            _subState = RState.End;
            _stopwatch.Restart();
            Notify("Start");
            return _runnerState;
        }
        public RState Start(string module, string name)
        {
            Reset();

            _module = module;
            _name = $"{name} (Token:{_RunnerToken++})";
            _runnerState = RState.Running;
            _subState = RState.End;
            _stopwatch.Restart();

            Notify("Start");
            return _runnerState;
        }
        /// <summary>
        /// 重试
        /// </summary>
        /// <param name="id"></param>
        /// <returns></returns>
        public RState Retry(Enum id,List<Enum> _preStepIds,string module, string name)
        {
            Reset();
            foreach (Enum item in _preStepIds)
            {
                _steps.Push(item);
            }
            _module = module;
            _name = $"{name} (Token:{_RunnerToken++})"; 
            _runnerState = RState.Running;
            _subState = RState.End;
            _stopwatch.Restart();
            Notify($"{id} Retry");
            _curStep = id;
            return _runnerState;
        }
        public void Stop(string reason,bool error=false)
        {
            if (Status == RState.Running)
            {
                Alarm(reason, error);
            }
            _runnerState = _subState = RState.Failed;
            _stopwatch.Stop();
        }

        public RoutineRunner Run(Enum id, Func<bool> action,  Func<bool> condition, int timeout = _defaultTimeout,bool error=true)
        {
            if (bAllowSubStepStart(id))
            {
                startNewStep(id, action,error);
            }
            else if(IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if(_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    Alarm($"Step:{id} timeout",error);
                    _runnerState = _subState = RState.Timeout;
                    _subStepTimer.Reset();
                }
            }
            return this;
        }

        public RoutineRunner RunConditionSubRoutine(Enum id,Func<int> routeIndexAction, IRoutine[] routines, object[] objects)
        {
            if (bAllowSubStepStart(id))
            {
                StartNewStepConditionRoutine(id, routeIndexAction, routines, objects);
            }
            else if(IsSubStepRunning(id))
            {
                if(_subRoutine==null)
                {
                    _subState = RState.End;
                }
                else
                {
                    if (objects.Length > _subRoutineIndex && _subRoutineIndex != -1)
                    {
                        _subRoutine.Start(objects[_subRoutineIndex]);
                    }
                    else
                    {
                        _subRoutine.Start();

                    }
                    _subState = RState.End;
                }
            }
            return this;
        }

        public RoutineRunner WaitConditionSubRoutine(Enum id)
        {
            if(bAllowSubStepStart(id))
            {
                Notify($"Step: {id} start ---");
                _steps.Push(id);

                _curStep = id;
                _subState = RState.Running;
                _subStepTimer.Restart();
            }
            else if(IsSubStepRunning(id))
            {
                if(_subRoutine==null)
                {
                    _subState= RState.End;
                    _subRoutineIndex = -1;
                }
                else
                {
                    _subState = _subRoutine.Monitor();
                    if(_subState==RState.End||_subState==RState.Failed||_subState==RState.Timeout)
                    {
                        _subRoutine = null;
                        _subRoutineIndex = -1;
                    }
                }
            }
            return this;
        }

        public void StartNewStepConditionRoutine(Enum id,Func<int> routeIndexAction, IRoutine[] routines, object[] objects)
        {
            _subRoutineIndex = routeIndexAction();
            if (routines.Length>_subRoutineIndex&&_subRoutineIndex!=-1)
            {
                _subRoutine = routines[_subRoutineIndex];
            }
            else
            {
                _subRoutine = null;
            }
            Notify($"Step: {id} start ---");
            _steps.Push(id);

            _curStep = id;
            _subState = RState.Running;
            _subStepTimer.Restart();
        }

        public RoutineRunner Run(Enum id, Func<bool> action, Func<bool> condition,Func<bool> failedCondition,bool skipToNext=false, int timeout = _defaultTimeout,bool error=true)
        {
            if (bAllowSubStepStart(id))
            {
                startNewStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if(failedCondition())
                {
                    if (!skipToNext)
                    {
                        _runnerState = _subState = RState.Failed;
                    }
                    else
                    {
                        _subState = RState.End;
                    }
                }
                else
                {
                    if (_subStepTimer.ElapsedMilliseconds >= timeout)
                    {
                        Alarm($"Step:{id} timeout", error);
                        if (!skipToNext)
                        {
                            _runnerState = _subState = RState.Timeout;
                        }
                        else
                        {
                            _subState = RState.End;
                        }
                        _subStepTimer.Reset();
                    }
                }
            }

            return this;
        }

        public RoutineRunner RunIf(Enum id, bool bRun, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if(bRun)
            {
                return Run(id, action, condition, timeout);
            }

            return this;
        }
        public RoutineRunner RunIf(Enum id, bool bRun, Func<bool> action, Func<bool> condition, Func<bool> failedCondition,bool skipToNext=false, int timeout = _defaultTimeout)
        {
            if (bRun)
            {
                return Run(id, action, condition, failedCondition,skipToNext,timeout);
            }

            return this;
        }
        public RoutineRunner Run(Enum id, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowSubStepStart(id))
            {
                startNewStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    _subState = RState.End;
                }
            }

            return this;
        }

        public RoutineRunner RunDelay(Enum id, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowSubStepStart(id))
            {
                startNewStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    if (!action())
                    {
                        Alarm($"Step: {id} Failed ");
                        _runnerState = _subState = RState.Failed;
                        _subStepTimer.Reset();
                    }
                    else
                    {
                        _subState = RState.End;
                    }

                }
            }

            return this;
        }


        public RoutineRunner RunIf(Enum id, bool bRun, Func<bool> action, int delayMS = _defaultDelay)
        {
            if(bRun)
            {
                return Run(id, action, delayMS);
            }

            return this;
        }


        public RoutineRunner End(Enum id, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (bAllowSubStepStart(id))
            {
                startNewStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    Notify($"End");
                    _runnerState = RState.End;
                    _subState = RState.End;
                }
                else
                {
                    if (_subStepTimer.ElapsedMilliseconds > timeout)
                    {
                        Alarm($"Step:{id} timeout");
                        _runnerState = RState.Failed;
                        _subState = RState.Timeout;
                    }
                }
            }

            return this;
        }

        public RoutineRunner End(Enum id, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowSubStepStart(id))
            {
                startNewStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    Notify($"End");
                    _runnerState = RState.End;
                    _subState = RState.End;
                    _stopwatch.Stop();
                }
            }

            return this;
        }

        public RoutineRunner Delay(Enum id, int delayMS = _defaultDelay_5s)
        {
            return Run(id, () => { return true; }, delayMS);
        }
        public RoutineRunner DelayIf(Enum id, bool bRun, int delayMS = _defaultDelay_5s)
        {
            if (bRun)
            {
                return Run(id, () => { return true; }, delayMS);
            }
            return this;
        }

        public RoutineRunner Wait(Enum id, Func<bool> condition, int timeout = _defaultTimeout)
        {
            return Run(id, () => { return true; }, condition, timeout);
        }
        public RoutineRunner WaitIf (Enum id, bool bRun, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (bRun)
            {
                return Run(id, () => { return true; }, condition, timeout);
            }
            return this;
        }

        public RoutineRunner WaitWithStopCondition(Enum id, Func<bool> condition, Func<bool> stopCondition, bool skipToNext = false,int timeout=_defaultTimeout,bool error=true)
        {
            return Run(id, () => { return true; }, condition, stopCondition,skipToNext,timeout,error);
        }
        public RoutineRunner WaitWithStopConditionIf(Enum id, bool bRun, Func<bool> condition, Func<bool> stopCondition,bool skipToNext=false, int timeout = _defaultTimeout, bool error = true)
        {
            if (bRun)
            {
                return Run(id, () => { return true; }, condition, stopCondition,skipToNext, timeout, error);
            }
            return this;
        }
        public RoutineRunner LoopWait(Enum id,Func<bool> action,Func<bool> condition,Func<bool> fail)
        {
            if (bAllowSubStepStart(id))
            {
                startNewStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if(condition())
                {
                    _subState = RState.End;
                }
                else if(fail())
                {
                    Notify($"End");
                    _runnerState = RState.End;
                    _subState = RState.End;
                }
            }
            return this;
        }

        public RoutineRunner LoopStart(Enum id, string name, int cycleCount, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if(bAllowSubStepStart(id))
            {
                _loopName = name;
                _loopCounterSP = cycleCount;
                _loopCounter = 0;
                _isLoopMode = true;
                startNewLoopStep(id, action);
            }
            else if(bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, action);
            }
            else if(IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    Alarm($"Step:{id} timeout");
                    _runnerState = _subState = RState.Timeout;
                    _subStepTimer.Reset();
                }
            }
            return this;
        }

        public RoutineRunner LoopStart(Enum id, string name, int cycleCount, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowSubStepStart(id))
            {
                _loopName = name;
                _loopCounterSP = cycleCount;
                _loopCounter = 0;
                _isLoopMode = true;
                startNewLoopStep(id, action);
            }
            else if (bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    _subState = RState.End;
                }
            }
                
            return this;
        }

        public RoutineRunner LoopRun(Enum id, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    Alarm($"Step:{id} timeout");
                    _runnerState = _subState = RState.Timeout;
                    _subStepTimer.Reset();
                }
            }

            return this;
        }

        public RoutineRunner LoopRunIf(Enum id, bool bRun, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if(bRun)
            {
                return LoopRun(id, action, condition, timeout);
            }

            return this;
        }

        public RoutineRunner LoopRunIfWithStopStatus(Enum id, bool bRun,Func<bool> condition, Func<bool> failedCondition, int timeout = _defaultTimeout)
        {
            if (bRun)
            {
                if (bAllowLoopSubStepStart(id))
                {
                    startNewLoopStep(id, () => { return true; });
                }
                else if (IsSubStepRunning(id))
                {
                    if (condition())
                    {
                        _subState = RState.End;
                    }
                    else if (failedCondition())
                    {
                        _runnerState = _subState = RState.Failed;
                    }
                    else
                    {
                        if (_subStepTimer.ElapsedMilliseconds >= timeout)
                        {
                            Alarm($"Step:{id} timeout", true);
                            _runnerState = _subState = RState.Timeout;
                            _subStepTimer.Reset();
                        }
                    }
                }
            }
            return this;
        }

        public RoutineRunner LoopRun(Enum id, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    _subState = RState.End;
                }
            }
            return this;
        }

        public RoutineRunner LoopRunOnlyTimeOutFault(Enum id, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    if (!action())
                    {
                        Alarm($"{_loopCounter + 1}th [{_loopName}] Loop Step: {id} failed");
                        _runnerState = _subState = RState.Failed;
                        _subStepTimer.Reset();
                    }
                    else
                    {
                        _subState = RState.End;
                    }
                }
            }
            return this;
        }

        public RoutineRunner LoopRunIfOnlyTimeOutFault(Enum id, bool bRun, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bRun)
            {
                return LoopRunOnlyTimeOutFault(id, action, delayMS);
            }

            return this;
        }


        public RoutineRunner LoopRunIf(Enum id, bool bRun, Func<bool> action, int delayMS = _defaultDelay)
        {
            if(bRun)
            {
                return LoopRun(id, action, delayMS);
            }

            return this;
        }
        public RoutineRunner LoopRunWithStopStatus(Enum id, Func<bool> condition, Func<bool> failedCondition,bool skipToNext=false, int timeout = _defaultTimeout)
        {
            if (bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (failedCondition())
                {
                    if (!skipToNext)
                    {
                        _runnerState = _subState = RState.Failed;
                    }
                    else
                    {
                        _subState = RState.End;
                    }
                }
                else
                {
                    if (_subStepTimer.ElapsedMilliseconds >= timeout)
                    {
                        Alarm($"Step:{id} timeout", true);
                        if (!skipToNext)
                        {
                            _runnerState = _subState = RState.Timeout;
                        }
                        else
                        {
                            _subState = RState.End;
                        }
                        _subStepTimer.Reset();
                    }
                }
            }

            return this;
        }

        


        public RoutineRunner LoopEnd(Enum id, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    Notify($"{_loopCounter + 1}th [{_loopName}] Loop End");

                    _subState = RState.End;
                    _loopCounter++;
                    if (_loopCounter >= _loopCounterSP)
                    {
                        foreach (var lid in _loopSteps)
                            _steps.Push(lid);

                        _loopSteps.Clear();
                        _isLoopMode = false;
                    }
                }
                else if (_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    Alarm($"Step:{id} timeout");
                    _runnerState = _subState = RState.Timeout;
                    _subStepTimer.Reset();
                }
            }

            return this;
        }
        public RoutineRunner LoopEnd(Enum id, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowLoopSubStepStart(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    Notify($"{_loopCounter + 1}th [{_loopName}] Loop End");

                    _subState = RState.End;
                    _loopCounter++;
                    if (_loopCounter >= _loopCounterSP)
                    {
                        foreach (var lid in _loopSteps)
                            _steps.Push(lid);

                        _loopSteps.Clear();
                        _isLoopMode = false;
                    }
                }
            }

            return this;
        }

        public RoutineRunner LoopDelay(Enum id, int delayMS)
        {
            return LoopRun(id, () => { return true; }, delayMS);
        }
        public RoutineRunner LoopDelayIf(Enum id, bool bRun, int delayMS)
        {
            if (bRun)
            {
                return LoopRun(id, () => { return true; }, delayMS);
            }
            return this;
        }
        public RoutineRunner LoopRunDelay(Enum id, Func<bool> action, int delayMS)
        {
            return LoopRun(id, action, delayMS);
        }

        public RoutineRunner LoopWait(Enum id, Func<bool> condition, int timeout = _defaultTimeout)
        {
            return LoopRun(id, () => { return true; }, condition, timeout);
        }

        public RoutineRunner LoopWaitIf(Enum id, bool bRun, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (bRun)
            {
                return LoopRun(id, () => { return true; }, condition, timeout);
            }
            return this;
               
        }

        public RoutineRunner LoopRetryStart(Enum id,string name, int cycleCount, Func<bool> action, int delayMS = _defaultDelay)
        {
            if (bAllowSubStepStart(id))
            {
                _loopName = name;
                _loopCounterSP = cycleCount;
                _loopCounter = 0;
                _isLoopMode = true;
                _loopSteps.Clear();
                startNewLoopStep(id, action);
            }
            else if (bAllowLoopSubStepStart(id)&&!_loopSteps.Contains(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (_subStepTimer.ElapsedMilliseconds > delayMS)
                {
                    _subState = RState.End;
                }
            }

            return this;
        }

        public RoutineRunner LoopRetrySecondRun(Enum id, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }
            if (bAllowLoopSubStepStart(id)&&_loopCounter>=1&&_loopSteps.Count!=0&&!_loopSteps.Contains(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    Alarm($"Step:{id} timeout");
                    _runnerState = _subState = RState.Timeout;
                    _subStepTimer.Reset();
                }
            }

            return this;
        }
        public RoutineRunner LoopRetrySecondRunWithStopStatus(Enum id, Func<bool> condition, Func<bool> failedCondition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }
            if (bAllowLoopSubStepStart(id)&&_loopCounter>=1 && _loopSteps.Count != 0 && !_loopSteps.Contains(id))
            {
                startNewLoopStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (failedCondition())
                {
                    _runnerState = _subState = RState.Failed;
                }
                else
                {
                    if (_subStepTimer.ElapsedMilliseconds >= timeout)
                    {
                        Alarm($"Step:{id} timeout", true);
                        _runnerState = _subState = RState.Timeout;
                        _subStepTimer.Reset();
                    }
                }
            }

            return this;
        }
        public RoutineRunner LoopRetryRun(Enum id, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }
           
            if (bAllowLoopSubStepStart(id) && _loopSteps.Count != 0 && !_loopSteps.Contains(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    Alarm($"Step:{id} timeout");
                    _runnerState = _subState = RState.Timeout;
                    _subStepTimer.Reset();
                }
            }


            return this;
        }

        public RoutineRunner LoopRetryRunBack(Enum id, Func<bool> action, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }

            if (bAllowLoopSubStepStart(id) && _loopSteps.Count != 0 && !_loopSteps.Contains(id))
            {
                startNewLoopStep(id, action);
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    _subState = RState.End;
                    _loopCounter++;
                    if (_loopCounter <= _loopCounterSP)
                    {
                        _loopSteps.Clear();
                    }
                    _subStepTimer.Reset();
                }
            }


            return this;
        }


        public RoutineRunner LoopRetryWait(Enum id, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }
            return LoopRetryRun(id, () => { return true; }, condition, timeout);
        }


        public RoutineRunner LoopRetryWaitBack(Enum id, Func<bool> condition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }

            if (bAllowLoopSubStepStart(id) && _loopSteps.Count != 0 && !_loopSteps.Contains(id))
            {
                startNewLoopStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (_subStepTimer.ElapsedMilliseconds >= timeout)
                {
                    _subState = RState.End;
                    _loopCounter++;
                    if (_loopCounter <= _loopCounterSP)
                    {
                        _loopSteps.Clear();
                    }
                    _subStepTimer.Reset();
                }
            }


            return this;
        }

        public RoutineRunner LoopRetryDelay(Enum id, int delayMS)
        {
            return LoopRetryRun(id, () => { return true; },()=> { return true; }, delayMS);
        }

        public RoutineRunner LoopRetryRunWithStopStatus(Enum id, Func<bool> condition, Func<bool> failedCondition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }
            if (bAllowLoopSubStepStart(id) && _loopSteps.Count != 0 && !_loopSteps.Contains(id))
            {
                startNewLoopStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (failedCondition())
                {
                    _runnerState = _subState = RState.Failed;
                }
                else
                {
                    if (_subStepTimer.ElapsedMilliseconds >= timeout)
                    {
                        Alarm($"Step:{id} timeout", true);
                        _runnerState = _subState = RState.Timeout;
                        _subStepTimer.Reset();
                    }
                }
            }

            return this;
        }
        public RoutineRunner LoopRetryRunWithStopStatusBack(Enum id, Func<bool> condition, Func<bool> failedCondition, int timeout = _defaultTimeout)
        {
            if (_loopCounter > _loopCounterSP)
            {
                return this;
            }
            if (bAllowLoopSubStepStart(id) && _loopSteps.Count != 0 && !_loopSteps.Contains(id))
            {
                startNewLoopStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                if (condition())
                {
                    _subState = RState.End;
                }
                else if (failedCondition())
                {
                    _subState = RState.End;
                    _loopCounter++;
                    if (_loopCounter <= _loopCounterSP)
                    {
                        _loopSteps.Clear();
                    }
                    _subStepTimer.Reset();
                }
                else
                {
                    if (_subStepTimer.ElapsedMilliseconds >= timeout)
                    {
                        _subState = RState.End;
                        _loopCounter++;
                        if (_loopCounter <= _loopCounterSP)
                        {
                            _loopSteps.Clear();
                        }
                        _subStepTimer.Reset();
                    }
                }
            }

            return this;
        }

        public RoutineRunner LoopRetryEnd(Enum id, int delayMS = _defaultDelay)
        {
            if (bAllowLoopSubStepStart(id)&& _loopSteps.Count != 0 && !_loopSteps.Contains(id))
            {
                startNewLoopStep(id, () => { return true; });
            }
            else if (IsSubStepRunning(id))
            {
                foreach (var lid in _loopSteps)
                    _steps.Push(lid);
                if (_loopCounter > _loopCounterSP)
                {
                    Alarm($"Step:{id} Retry times over {_loopCounterSP}");
                    _runnerState = _subState = RState.Timeout;
                    _subStepTimer.Reset();
                }
                else
                {
                    _subState = RState.End;
                }
                _loopSteps.Clear();
                _isLoopMode = false;
            }

            return this;
        }

        private void startNewStep(Enum id, Func<bool> action,bool error=true)
        {
            if (action())
            {
                 Notify($"Step: {id} start ---");
                 _steps.Push(id);
                _curStep = id;
                _subState = RState.Running;
                _subStepTimer.Restart();
            }
            else
            {
                Alarm($"Step: {id} failed ",error);
                _runnerState = RState.Failed;
            }
        }



        private void startNewLoopStep(Enum id, Func<bool> action)
        {
            if (action())
            {
                Notify($"{_loopCounter + 1}th [{_loopName}] Loop Step: {id} start ---");
                if (!_loopSteps.Contains(id))
                    _loopSteps.Push(id);

                _curStep = id;
                _subState = RState.Running;
                _subStepTimer.Restart();
            }
            else
            {
                Alarm($"{_loopCounter + 1}th [{_loopName}] Loop Step: {id} failed");
                _runnerState = RState.Failed;
            }
        }

        protected void Notify(string message)
        {
            LOG.WriteBackgroundLog(eEvent.EV_ROUTINE_NOTIFY, _module, _name, message);
        }
        protected void Alarm(string message,bool error=true)
        {
            if (error)
            {
                _errorMsg= message;
                LOG.WriteLog(eEvent.ERR_ROUTINE_FAILED, _module,_name,message);
            }
            else
            {
                LOG.WriteLog(eEvent.EV_ROUTINE_NOTIFY, _module, _name,message);
            }
        }
    }
}