CyberX8/CyberX8_RT/Dispatch/JobProcesser.cs

using Aitex.Core.RT.Log;
using MECF.Framework.Common.Jobs;
using System;
using System.Collections.Generic;
using MECF.Framework.Common.Equipment;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Aitex.Core.Common;
using MECF.Framework.Common.SubstrateTrackings;
using MECF.Framework.Common.DBCore;
using Aitex.Core.RT.DataCenter;
using Aitex.Core.Util;
using CyberX8_RT.Modules;
using CyberX8_RT.Schedulers;
using Aitex.Core.RT.SCCore;
using MECF.Framework.Common.RecipeCenter;
using Aitex.Core.RT.RecipeCenter;
using Aitex.Common.Util;
using System.Collections.Concurrent;
using MECF.Framework.Common.WaferHolder;
using CyberX8_RT.Modules.Loader;
using SecsGem.Core.ItemModel;
using CyberX8_RT.Devices.EFEM;

namespace CyberX8_RT.Dispatch
{
    public class JobProcesser : Singleton<JobProcesser>
    {
        #region 常量
        private const string PRODUCTION = "Production";
        private const string SEQUENCE_TYPE = "SequenceType";
        #endregion

        #region 内部变量
        /// <summary>
        /// control jobs
        /// </summary>
        private List<ControlJobInfo> _lstControlJobs = new List<ControlJobInfo>();
        /// <summary>
        /// process jobs
        /// </summary>
        private List<ProcessJobInfo> _lstProcessJobs = new List<ProcessJobInfo>();
        private SchedulerFACallback _faCallback;
        private SchedulerDBCallback _dbCallback; 
        private Dictionary<string, ControlJobInfo> _loadportControlJobDic = new Dictionary<string, ControlJobInfo>();
        /// <summary>
        /// recipe processjob集合(key-recipe ppid,value-Process job集合)
        /// </summary>
        private ConcurrentDictionary<string, List<ProcessJobInfo>> _recipeProcessJobs = new ConcurrentDictionary<string, List<ProcessJobInfo>>();
        /// <summary>
        /// recipe集合
        /// </summary>
        private List<string> _recipeList = new List<string>();
        /// <summary>
        /// recipe锁
        /// </summary>
        private object _recipeLocker = new object();
        #endregion

        #region 属性
        /// <summary>
        /// ControlJob集合
        /// </summary>
        public List<ControlJobInfo> ControlJobInfos { get { return _lstControlJobs; } }
        /// <summary>
        /// ProcessJob集合
        /// </summary>
        public List<ProcessJobInfo> ProcessJobInfos { get { return _lstProcessJobs; } }
        #endregion

        /// <summary>
        /// 构造函数
        /// </summary>
        public JobProcesser()
        {
            _faCallback = new SchedulerFACallback();
            _dbCallback = new SchedulerDBCallback();
            InitializeData();
        }
        /// <summary>
        /// 初始化数据
        /// </summary>
        private void InitializeData()
        {
            for (int i = 1; i <= 3; i++)
            {
                _loadportControlJobDic[$"LP{i}"] = null;
                string lp = $"LP{i}";
                DATA.Subscribe($"{lp}.CurrentControlJob", () => _loadportControlJobDic[lp], SubscriptionAttribute.FLAG.IgnoreSaveDB);
            }
            DATA.Subscribe("Scheduler.CjNameList", () => Array.ConvertAll(_lstControlJobs.ToArray(), x => x.Name.ToString()).ToList(), SubscriptionAttribute.FLAG.IgnoreSaveDB);

        }

        /// <summary>
        /// 创建任务
        /// </summary>
        /// <param name="param"></param>
        /// <param name="reason"></param>
        /// <returns></returns>
        public bool CreateJob(Dictionary<string, object> param, out string reason)
        {
            reason = "";
            string[] slotSequence = (string[])param["SlotSequence"];
            string jobId = (string)param["JobId"];
            string module = (string)param["Module"];

            if (string.IsNullOrEmpty(jobId))
            {
                jobId = "CJ_Local_" + module;
            }
            string lotId = jobId;
            if (param.ContainsKey("LotId"))
                lotId = (string)param["LotId"];

            int cycleNumber = param.ContainsKey("CycleNumber") ? (int)param["CycleNumber"] : 1;
            if (slotSequence.Length != SC.GetValue<int>("EFEM.LoadPort.SlotNumber"))
            {
                reason = $"slot sequence parameter not valid, length is {slotSequence.Length}, should be {SC.GetValue<int>("EFEM.LoadPort.SlotNumber")}";
                LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                return false;
            }

            string sequenceType = PRODUCTION;
            if (param.ContainsKey(SEQUENCE_TYPE))
            {
                sequenceType = param[SEQUENCE_TYPE].ToString();
            }
            ModuleName lpModule = ModuleHelper.Converter(module);
            if (!ModuleHelper.IsLoadPort(lpModule))
            {
                reason = $"{module} should be LoadPort";
                LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                return false;
            }

            if (_lstControlJobs.Exists(x => x.Name == jobId))
            {
                reason = $"{jobId} already created";
                LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                return false;
            }


            ControlJobInfo cj = new ControlJobInfo();
            cj.Name = jobId;
            cj.Module = module;
            cj.LotName = lotId;
            cj.LotInnerId = Guid.NewGuid();
            cj.LotWafers = new List<WaferInfo>();
            cj.SetState(EnumControlJobState.WaitingForStart);
            cj.SequenceNameList = slotSequence;
            cj.CycleNumber = cycleNumber;

            Dictionary<string, bool[]> seqSlot = new Dictionary<string, bool[]>();
            Dictionary<string, List<Tuple<ModuleName, int>>> seqSlotWafers = new Dictionary<string, List<Tuple<ModuleName, int>>>();
            Dictionary<string, string> indexSequence = new Dictionary<string, string>();

            bool enableGroupBySequence = SC.GetValue<bool>("Scheduler.GroupWaferBySequence");

            int waferCount = 0;
            for (int i = 0; i < SC.GetValue<int>("EFEM.LoadPort.SlotNumber"); i++)
            {
                if (string.IsNullOrEmpty(slotSequence[i]) || string.IsNullOrEmpty(slotSequence[i].Trim()))
                    continue;

                string groupName = enableGroupBySequence ? slotSequence[i].Trim() : i.ToString();
                indexSequence[groupName] = slotSequence[i];
                if (!seqSlot.ContainsKey(groupName))
                {
                    seqSlot[groupName] = new bool[SC.GetValue<int>("EFEM.LoadPort.SlotNumber")];
                }
                if (!seqSlotWafers.ContainsKey(groupName))
                {
                    seqSlotWafers[groupName] = new List<Tuple<ModuleName, int>>();
                }
                seqSlot[groupName][i] = true;

                if (!WaferManager.Instance.CheckHasWafer(module, i))
                {
                    reason = $"job wafer: {module} slot {i + 1} not in the carrier";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }
                if (!WaferManager.Instance.CheckWafer(ModuleHelper.Converter(module), i, WaferStatus.Normal))
                {
                    reason = $"job wafer: {module} slot {i + 1} status is {WaferManager.Instance.GetWafer(ModuleHelper.Converter(module), i).Status}";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }

                if (WaferManager.Instance.GetWafer(ModuleHelper.Converter(module), i).ProcessState != EnumWaferProcessStatus.Idle)
                {
                    reason = $"job wafer: {module} slot {i + 1} process status is {WaferManager.Instance.GetWafer(ModuleHelper.Converter(module), i).ProcessState}";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }
                //--- 2024-03-21 增加了waferinfo 相应的lotId信息 start---
                WaferInfo waferInfo = WaferManager.Instance.GetWafer(ModuleHelper.Converter(module), i);
                cj.LotWafers.Add(waferInfo);
                waferInfo.SequenceName = slotSequence[i];
                waferInfo.LotId = lotId;
                //--- 2024-03-21 增加了waferinfo 相应的lotId信息 end---

                seqSlotWafers[groupName].Add(Tuple.Create(ModuleHelper.Converter(module), i));

                cj.JobWaferSize = WaferManager.Instance.GetWafer(ModuleHelper.Converter(module), i).Size;
                waferCount++;
                LOG.WriteLog(eEvent.EV_ROUTER, ModuleName.System.ToString(), $"Assigned wafer job, wafer {module}.{i + 1}, sequence: {slotSequence[i]}");
            }

            if (seqSlotWafers.Count == 0)
            {
                reason = $"job has not assign wafer";
                LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                return false;
            }

            //AutoIdle
            if (Singleton<RouteManager>.Instance.IsAutoIdle)
            {
                bool waferCacheCheck = WaferManager.Instance.CheckStartUpCache(ref reason);
                if (!waferCacheCheck)
                {
                    return true;
                }

                bool waferHolderCacheCheck = WaferHolderManager.Instance.CheckStartUpWaferHolderBuffer(ref reason);
                if (!waferHolderCacheCheck)
                {
                    return false;
                }


                LoaderEntity loaderEntity = Singleton<RouteManager>.Instance.GetModule<LoaderEntity>(ModuleName.Loader1.ToString());
                if (loaderEntity != null && loaderEntity.WaferHolderInfo != null)
                {
                    reason = "Loader Has WaferHolder";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }

                UpdateAllBufferCellWaferHollderToLoader();
            }
            List<ProcessJobInfo> pjs = new List<ProcessJobInfo>();
            string[] seqs = seqSlot.Keys.ToArray();
            for (int i = 0; i < seqs.Length; i++)
            {
                ProcessJobInfo pj = new ProcessJobInfo();
                pj.Name = jobId + "_" + (i + 1);
                pj.ControlJobName = cj.Name;
                pj.LotName = lotId;
                pj.SlotWafers = seqSlotWafers[seqs[i]];
                pj.SetState(EnumProcessJobState.Queued);

                WaferSize slotWafersSize = GetWaferSize(pj.SlotWafers);
                string sequenceDirectory = sequenceType == PRODUCTION ? PathManager.GetProductionRecipeDir():PathManager.GetEngineeringRecipeDir();
                string recipeFullPath = sequenceDirectory + $"\\{indexSequence[seqs[i]]}.seq.rcp";
                SequenceRecipe sequenceRecipe = RecipeFileManager.Instance.LoadGenericityRecipe<SequenceRecipe>(recipeFullPath);
                if (sequenceRecipe==null)
                {
                    reason = $"sequence recipe file {pj.Sequence} is not exist";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }
                sequenceRecipe.SequenceType = sequenceType;
                if (!SchedulerSequenceRecipeManager.Instance.CheckSequenceRecipeAvaible(sequenceRecipe,ref reason))
                {
                    return false;
                }
                if(!SchedulerSequenceRecipeManager.Instance.ExistAvaibleProcessCell(sequenceRecipe,true,true))
                {
                    reason = "sequence meets no avaible cell";
                    return false;
                }
                pj.SequenceRecipe= sequenceRecipe;


                WaferSize waferSize = (WaferSize)pj.SequenceRecipe.SubstrateSize;

                if (slotWafersSize!= waferSize)
                {
                    reason = $"sequence wafersize {pj.SequenceRecipe.SubstrateSize} is not matched wafer wafersize {slotWafersSize}";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }

                int sideAWaferSize = SC.GetValue<int>($"Loader1.SideAWaferSize");
                int sideBWaferSize = SC.GetValue<int>($"Loader1.SideBWaferSize");
                if ((int)waferSize != sideAWaferSize && (int)waferSize != sideBWaferSize)
                {
                    reason = $"loader is not matched wafer wafersize {waferSize}";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }
                int dummyCount = 0;
                DummyDevice dummy1Device = Singleton<RouteManager>.Instance.EFEM.GetDummyDevice(0);
                if(dummy1Device!=null&&dummy1Device.HasCassette&&dummy1Device.WaferSize==waferSize)
                {
                    dummyCount++;
                }
                DummyDevice dummy2Device = Singleton<RouteManager>.Instance.EFEM.GetDummyDevice(1);
                if (dummy2Device != null && dummy2Device.HasCassette && dummy2Device.WaferSize == waferSize)
                {
                    dummyCount++;
                }
                if (dummyCount == 0)
                {
                    reason = $"dummy is not matched wafer wafersize {waferSize}";
                    LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                    return false;
                }
                string crsType = pj.SequenceRecipe.CrsType;
                List<string> chemistries = SchedulerSequenceRecipeManager.Instance.GetSequenceChemistry(pj.SequenceRecipe);
                if (ControlJobInfos.Count == 0)
                {
                    bool avaibleWaferHolder= SchedulerManager.Instance.ExistAvaibleBufferWaferHolder(waferSize, crsType, chemistries);
                    if (!avaibleWaferHolder)
                    {
                        reason = $"buffer has not enough wafer shuttle";
                        LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                        return false;
                    }
                }
                else
                {
                    int waferHolderCount = SchedulerManager.Instance.GetWaferHolderInfoCountByWaferSize(waferSize, crsType, chemistries);
                    if (waferHolderCount == 0)
                    {
                        reason = $"buffer has not enough wafer shuttle";
                        LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                        return false;
                    }
                }
                pjs.Add(pj);
            }

            _dbCallback.LotUpdate(cj);
            foreach (var pj in pjs)
            {
                cj.ProcessJobNameList.Add(pj.Name);
                _lstProcessJobs.Add(pj);
            }

            _lstControlJobs.Add(cj);
            //AssociatedPMWithLP(cj);
            _loadportControlJobDic[cj.Module] = cj;
            CycleManager.Instance.UpdateLoadportSp(cj.Module,cycleNumber);
            return true;
        }
        /// <summary>
        /// 获取Wafer尺寸
        /// </summary>
        /// <param name="slotWafers"></param>
        /// <returns></returns>
        private WaferSize GetWaferSize(List<Tuple<ModuleName,int>> slotWafers)
        {
            var result = slotWafers[0];
            WaferInfo waferInfo = WaferManager.Instance.GetWafer(result.Item1, result.Item2);
            if (!waferInfo.IsEmpty)
            {
                return waferInfo.Size;
            }
            return WaferSize.WS0;
        }
        /// <summary>
        /// 更新所有BufferCellWaferHolder可以拉至Loader
        /// </summary>
        private void UpdateAllBufferCellWaferHollderToLoader()
        {
            List<SchedulerModule> schedulerModules = SchedulerManager.Instance.GetModuleTypeScheduleModules(ModuleType.Buffer);
            foreach (SchedulerModule item in schedulerModules)
            {
                WaferHolderInfo waferHolderInfo = WaferHolderManager.Instance.GetWaferHolder(item.Module.ToString());
                if (waferHolderInfo != null)
                {
                    waferHolderInfo.IsToLoader = true;
                }
            }
        }
        /// <summary>
        /// 增加processjob至字典中
        /// </summary>
        /// <param name="processJobInfo"></param>
        public void AddRecipeProcessJob(ProcessJobInfo processJobInfo)
        {
            List<ProcessJobInfo> jobs = null;
            if(_recipeProcessJobs.ContainsKey(processJobInfo.SequenceRecipe.Ppid))
            {
                jobs = _recipeProcessJobs[processJobInfo.SequenceRecipe.Ppid];
            }
            else
            {
                jobs = new List<ProcessJobInfo>();
                _recipeProcessJobs[processJobInfo.SequenceRecipe.Ppid]=jobs;
            }
            jobs.Add(processJobInfo);
            lock (_recipeLocker)
            {
                if(!_recipeList.Contains(processJobInfo.SequenceRecipe.Ppid))
                {
                    _recipeList.Add(processJobInfo.SequenceRecipe.Ppid);
                }
            }
        }
        /// <summary>
        /// 获取启动recipe的Processjob集合
        /// </summary>
        /// <returns></returns>
        public List<ProcessJobInfo> GetRecipeStateProcessJobs(EnumProcessJobState enumProcessJobState)
        {
            string recipe = "";
            lock (_recipeLocker)
            {
                if (_recipeList.Count != 0)
                {
                    recipe = _recipeList[0];
                }
            }
            List<ProcessJobInfo> processJobInfos = new List<ProcessJobInfo>();
            if(!string.IsNullOrEmpty(recipe)&&_recipeProcessJobs.ContainsKey(recipe))
            {
                List<ProcessJobInfo> recipeProcessJobs = _recipeProcessJobs[recipe];
                foreach (ProcessJobInfo processJobInfo in recipeProcessJobs)
                {
                    if (processJobInfo.State == enumProcessJobState)
                    {
                        processJobInfos.Add(processJobInfo);
                    }
                }
            }
            return processJobInfos;
        }
        /// <summary>
        /// 移除recipe processjob字典中processjob
        /// </summary>
        /// <param name="processJobInfo"></param>
        private void RemoveRecipeProcessJob(ProcessJobInfo processJobInfo)
        {
            List<ProcessJobInfo> jobs = null;
            if (_recipeProcessJobs.ContainsKey(processJobInfo.SequenceRecipe.Ppid))
            {
                jobs = _recipeProcessJobs[processJobInfo.SequenceRecipe.Ppid];
                if(jobs.FindIndex(O => O.InnerId == processJobInfo.InnerId)>=0)
                {
                    jobs.Remove(processJobInfo);
                }
                if (jobs.Count == 0)
                {
                    _recipeProcessJobs.TryRemove(processJobInfo.SequenceRecipe.Ppid, out jobs);
                    lock (_recipeLocker)
                    {
                        _recipeList.Remove(processJobInfo.SequenceRecipe.Ppid);
                    }
                }
            }
        }
        /// <summary>
        /// 暂停Job
        /// </summary>
        /// <param name="jobName"></param>
        /// <param name="reason"></param>
        /// <returns></returns>
        public bool PauseJob(string jobName, out string reason)
        {
            reason = "";
            ControlJobInfo cj = _lstControlJobs.Find(x => x.Name == jobName);
            if (cj == null)
            {
                reason = $"pause job rejected, not found job with id {jobName}";
                return false;
            }

            if (cj.State == EnumControlJobState.Executing)
            {
                cj.SetState(EnumControlJobState.Paused);
            }
            _faCallback.JobPaused(cj);

            return true;
        }

        /// <summary>
        /// 恢复Job
        /// </summary>
        /// <param name="jobName"></param>
        /// <param name="reason"></param>
        /// <returns></returns>
        public bool ResumeJob(string jobName, out string reason)
        {
            reason = "";
            ControlJobInfo cj = _lstControlJobs.Find(x => x.Name == jobName);
            if (cj == null)
            {
                reason = $"resume job rejected, not found job with id {jobName}";
                //LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System, $"resume job rejected, not found job with id {jobName}");
                return false;
            }

            if (cj.State == EnumControlJobState.Paused)
            {
                cj.SetState(EnumControlJobState.Executing);
            }

            _faCallback.JobResumed(cj);
            return true;
        }
        /// <summary>
        /// 启动任务
        /// </summary>
        /// <param name="jobName"></param>
        /// <param name="reason"></param>
        /// <returns></returns>
        public bool StartJob(string jobName, out string reason)
        {
            reason = "";
            ControlJobInfo cj = _lstControlJobs.Find(x => x.Name == jobName);
            if (cj == null)
            {
                reason = $"start job rejected, not found job with id {jobName}";
                LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                return false;
            }

            if (cj.State == EnumControlJobState.WaitingForStart)
            {
                cj.SetState(EnumControlJobState.Executing);
                cj.StartTime = DateTime.Now;

                _dbCallback.LotCreated(cj);
                _faCallback.JobStarted(cj);
                CycleManager.Instance.InitializeLoadPortData(cj.Module);
                foreach(string item in cj.ProcessJobNameList)
                {
                    ProcessJobInfo pj=_lstProcessJobs.Find(O=>O.Name== item);
                    if(pj !=null)
                    {
                        AddRecipeProcessJob(pj);
                    }
                }
            }
            return true;
        }

        /// <summary>
        /// 停止任务
        /// </summary>
        /// <param name="jobName"></param>
        /// <param name="reason"></param>
        /// <returns></returns>
        public bool StopJob(string jobName, out string reason)
        {
            reason = "";
            ControlJobInfo cj = _lstControlJobs.Find(x => x.Name == jobName);
            if (cj == null)
            {
                reason = $"stop job rejected, not found job with id {jobName}";
                LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), reason);
                return false;
            }
            if (cj.CycleNumber > 1)
            {
                cj.CycleNumber = 1;
                LOG.WriteLog(eEvent.WARN_ROUTER, ModuleName.System.ToString(), $"stop controljob {cj.Name} cycle number changed to 1");
            }
            int processJobCount = 0;
            foreach (var pj in _lstProcessJobs)
            {
                if (pj.ControlJobName == cj.Name)
                {
                    pj.SetState(EnumProcessJobState.Stopping);
                    int count = 0;
                    foreach (var pjSlotWafer in pj.SlotWafers)
                    {
                        if (WaferManager.Instance.CheckHasWafer(pjSlotWafer.Item1, pjSlotWafer.Item2))
                        {
                            WaferInfo wafer = WaferManager.Instance.GetWafer(pjSlotWafer.Item1, pjSlotWafer.Item2);
                            if (WaferTaskManager.Instance.Contains(wafer.WaferID))
                            {
                                continue;
                            }
                            if (wafer.ProcessState == EnumWaferProcessStatus.Idle)
                            {
                                wafer.ProcessState = EnumWaferProcessStatus.Canceled; 
                                count++;
                                LOG.WriteLog(eEvent.EV_SCHEDULER, "System", $"{pjSlotWafer.Item1}.{pjSlotWafer.Item2} cancel process");
                            }
                            
                        }                        
                    }
                    if (count == pj.SlotWafers.Count)
                    {
                        processJobCount++;
                    }
                }
            }

            if (processJobCount == _lstProcessJobs.Count)
            {
                List<ProcessJobInfo> lst = _lstProcessJobs.ToList();
                foreach (var pj in lst)
                {
                    _lstProcessJobs.Remove(pj);
                    RemoveRecipeProcessJob(pj); 
                    LOG.WriteLog(eEvent.EV_SCHEDULER, "System", $"controljob {cj.Name} processjob {pj.SequenceRecipe.Ppid} removed");
                }
            }

            if (processJobCount == _lstProcessJobs.Count)
            {
                ClearLoadPortControlJob(cj);

                _lstControlJobs.Remove(cj);
                _faCallback.JobStopped(cj);
                _dbCallback.LotFinished(cj);
                LOG.WriteLog(eEvent.EV_SCHEDULER, "System", $"controljob {cj.Name} removed");
            }
            return true;
        }
        /// <summary>
        /// 获取正在执行的ControlJob
        /// </summary>
        /// <returns></returns>
        public ControlJobInfo GetRunningControlJob()
        {
            return _lstControlJobs.Find(O => O.State == EnumControlJobState.Executing);
        }
        /// <summary>
        /// 获取正在执行的ControlJob集合
        /// </summary>
        /// <returns></returns>
        public List<ControlJobInfo> getRunningControlJobs()
        {
            return _lstControlJobs.FindAll(O => O.State == EnumControlJobState.Executing);
        }
        /// <summary>
        /// ControlJob任务完成
        /// </summary>
        public void CompleteControlJob(ControlJobInfo controlJobInfo)
        {
            controlJobInfo.SetState(EnumControlJobState.Completed);
            controlJobInfo.EndTime = DateTime.Now;
            _dbCallback.LotFinished(controlJobInfo);
            _faCallback.JobFinished(controlJobInfo);

            List<ProcessJobInfo> pjRemoveList = new List<ProcessJobInfo>();
            foreach (var pj in _lstProcessJobs)
            {
                if (pj.ControlJobName == controlJobInfo.Name)
                    pjRemoveList.Add(pj);
            }

            foreach (var pj in pjRemoveList)
            {
                _lstProcessJobs.Remove(pj);
                RemoveRecipeProcessJob(pj);
            }

            ClearLoadPortControlJob(controlJobInfo);

            _lstControlJobs.Remove(controlJobInfo);
        }
        /// <summary>
        /// 清除LoadPort ControlJob
        /// </summary>
        /// <param name="controlJobInfo"></param>
        private void ClearLoadPortControlJob(ControlJobInfo controlJobInfo)
        {
            string[] keys = _loadportControlJobDic.Keys.ToArray();
            foreach(string key in  keys)
            {
                ControlJobInfo tmp= _loadportControlJobDic[key];
                if(tmp!=null&&tmp.Name==controlJobInfo.Name)
                {
                    _loadportControlJobDic[key] = null;
                }
            }
        }
        /// <summary>
        /// 重新启动任务
        /// </summary>
        /// <param name="controlJobInfo"></param>
        public void RestartControlJob(ControlJobInfo controlJobInfo)
        {
            List<ProcessJobInfo> pjRemoveList = new List<ProcessJobInfo>();
            foreach (var pj in _lstProcessJobs)
            {
                if (pj.ControlJobName == controlJobInfo.Name)
                    pjRemoveList.Add(pj);
            }

            foreach (var pj in pjRemoveList)
            {
                RemoveRecipeProcessJob(pj);
            }
            pjRemoveList.Clear();

            controlJobInfo.SetState(EnumControlJobState.Executing);
            controlJobInfo.InnerId= Guid.NewGuid();
            controlJobInfo.LotInnerId= Guid.NewGuid();
            _dbCallback.LotCreated(controlJobInfo);
            foreach (var pj in _lstProcessJobs)
            {
                if (pj.ControlJobName == controlJobInfo.Name)
                {
                    pj.SetState(EnumProcessJobState.Queued);
                    pj.InnerId = Guid.NewGuid();
                    foreach (var pjSlotWafer in pj.SlotWafers)
                    {
                        WaferInfo wafer = WaferManager.Instance.GetWafer(pjSlotWafer.Item1, pjSlotWafer.Item2);
                        wafer.ProcessJob = null;
                        wafer.NextSequenceStep = 0;
                        wafer.ProcessState = EnumWaferProcessStatus.Idle;
                    }

                    AddRecipeProcessJob(pj);
                }
            }
        }
        /// <summary>
        /// 获取Job对象
        /// </summary>
        /// <param name="name"></param>
        /// <returns></returns>
        public ControlJobInfo GetControlJobInfoByName(string name)
        {
            return _lstControlJobs.Find(O=>O.Name == name);
        }
        /// <summary>
        /// 根据ProcessJob获取ControlJob
        /// </summary>
        /// <param name="processJobInfo"></param>
        /// <returns></returns>
        public ControlJobInfo GetControlJobInfoByProcessJob(ProcessJobInfo processJobInfo)
        {
            return _lstControlJobs.Find(O => O.Name == processJobInfo.ControlJobName);
        }
        /// <summary>
        /// System Abort
        /// </summary>
        public void SystemAbort()
        {
            List<ControlJobInfo> controlJobInfos = _lstControlJobs.ToList();
            foreach(var item in controlJobInfos)
            {
                StopJob(item.Name, out string reason);
            }
        }
        /// <summary>
        /// 移除所有job
        /// </summary>
        public void RemoveAllJob()
        {
            List<ControlJobInfo> controlJobInfos = _lstControlJobs.ToList();
            foreach (var item in controlJobInfos)
            {
                List<ProcessJobInfo> pjRemoveList = new List<ProcessJobInfo>();
                foreach (var pj in _lstProcessJobs)
                {
                    if (pj.ControlJobName == item.Name)
                        pjRemoveList.Add(pj);
                }

                foreach (var pj in pjRemoveList)
                {
                    _lstProcessJobs.Remove(pj);
                    RemoveRecipeProcessJob(pj);
                }

                ClearLoadPortControlJob(item);

                _lstControlJobs.Remove(item);
            }
            WaferTaskManager.Instance.RemoveAllTask();
            WaferHolderTaskManager.Instance.RemoveAllTask();
            SchedulerManager.Instance.ResetSchedulerModule();
        }
        /// <summary>
        /// 获取loadport的controljob
        /// </summary>
        /// <param name="loadPort"></param>
        /// <returns></returns>
        public ControlJobInfo GetControlJobInfoByLoadPort(string loadPort)
        {
            return _loadportControlJobDic.ContainsKey(loadPort)?_loadportControlJobDic[loadPort]:null;
        }
    }
}