Optimizer.h

#pragma once
 
#if _MSC_VER >= 1900 && defined(_M_X64)
 
#include "DefinitionsAI.h"
#include "Tensor.h"
 
namespace FLImaging
{
    namespace AI
    {
        template <typename T>
        class CInternalOptimizerBase;
 
        template <typename T>
        class CComputationalGraph;
 
        template <typename T>
        class CComputationalGraphPlaceholder;
 
        template <typename T>
        class CComputationalGraphAugmentationBase;
 
        template <typename T>
        class CTensor;
 
        template <typename T>
        class CLearningRateSchedulerBase;
 
        template <typename T>
        class CClassEqualizerBase;
 
        template <typename T>
        class CValidatorBase;
 
        template <typename T>
        class CInternalOptimizerSpecBase;
 
        template <typename T>
        class FL_EXPORT COptimizer : public CAlgorithmAIBase
        {
        public:
            COptimizer();
            COptimizer(const COptimizer<T>& otmz);
            COptimizer(const COptimizer<T>* pOtmz);
            virtual ~COptimizer();
 
            const COptimizer<T>& operator=(const COptimizer<T>& otmz);
 
            virtual const CResult Clear();
            virtual const CResult ClearMemory();
            virtual const CResult ClearHistories();
 
            virtual const CResult Assign(const COptimizer<T>& ob);
            virtual const CResult Assign(const COptimizer<T>* pOb);
 
            virtual const CResult Initialize();
            virtual const CResult InitializeBatch();
            virtual const CResult InitializeMiniBatch();
            virtual bool IsInitialized() const;
 
            virtual const CResult SetFunction(CComputationalGraph<T>& cgFunction);
            virtual const CResult SetFunction(CComputationalGraph<T>* pCgFunction);
            virtual CComputationalGraph<T>& GetFunction();
 
            virtual const CResult SetGPUIndices(const std::vector<int32_t>& vctGPUIndices);
            virtual const CResult SetGPUIndices(const Base::CFLArray<int32_t>& flaGPUIndices);
            virtual const CResult SetGPUIndices(const Base::CFLArray<uint32_t>& flaGPUIndices);
            virtual const std::vector<int32_t>& GetGPUIndices() const;
 
            virtual const CResult SetInternalOptimizerSpec(CInternalOptimizerSpecBase<T>* pInternalOptimizerSpec);
            virtual const CInternalOptimizerSpecBase<T>* GetInternalOptimizerSpec() const;
 
            virtual const CResult SetLearningRateScheduler(CLearningRateSchedulerBase<T>& lrScheduler);
            virtual const CResult SetLearningRateScheduler(CLearningRateSchedulerBase<T>* pLrScheduler);
            virtual const CLearningRateSchedulerBase<T>* GetLearningRateScheduler() const;
 
            virtual const CResult EnableClassEqualization(bool bEnable);
            virtual bool IsClassEqualizationEnabled() const;
 
            virtual const CResult SetClassEqualizer(CClassEqualizerBase<T>& ce);
            virtual const CResult SetClassEqualizer(CClassEqualizerBase<T>* pCe);
            virtual const CClassEqualizerBase<T>* GetClassEqualizer() const;
 
            virtual const CResult SetValidator(CValidatorBase<T>& validator);
            virtual const CResult SetValidator(CValidatorBase<T>* pValidator);
            virtual const CValidatorBase<T>* GetValidator() const;
 
            virtual const std::vector<int64_t>& GetValidationIndices() const;
 
            virtual const CResult SetLearningRateSchedulerMode(ELearningRateSchedulerMode eLrsMode = ELearningRateSchedulerMode_Epoch);
            virtual ELearningRateSchedulerMode GetLearningRateSchedulerMode() const;
 
            virtual const CResult ReducedWeightGradients();
            virtual const CResult PropagateWeights();
 
            virtual const CResult ResetDerivatives();
 
            virtual const CResult EnableResetDerivatives(bool bResetDerivatives);
            virtual bool IsResetDerivativesEnabled() const;
 
            virtual int64_t GetTotalIteration() const;
            virtual int64_t GetSubdivisionCount() const;
            virtual int64_t GetMiniBatchSizePerSubdivision() const;
            virtual int64_t GetTotalBatchSize() const;
            virtual int64_t GetActualTotalBatchSize() const;
            virtual int64_t GetValidationTotalIteration() const;
            virtual int64_t GetCurrentEpoch() const;
 
            virtual const CResult Fit(T* pClippingThreshold = nullptr, int32_t* pI32IterationFeedback = nullptr, int32_t* pI32CumulativeIterationFeedback = nullptr, Base::CFLArray<int32_t>* pFlaSubdivisionFeedback = nullptr, bool* pBStopImmediately = nullptr, int64_t* pI64MeanTimePerMiniBatch = nullptr, int64_t* pI64LastTimeOfMiniBatch = nullptr);
            virtual T Validate(Base::CFLArray<int32_t>* pFlaIteration = nullptr);
            virtual const CTensor<T>& GetResult() const;
 
            virtual const std::vector<T>& GetLearningResultAccuracyHistory() const;
            virtual T GetLearningResultLastAccuracyValue() const;
 
            virtual const Base::CFLArray<uint32_t> GetInvolvedDeviceIndices() const;
            virtual const CInternalOptimizerBase<T>* GetInternalOptimizer() const;
 
            virtual const CResult SetSubdivisionMultiplier(int64_t i64SubdivisionMultiplier = 1);
            virtual int64_t GetSubdivisionMultiplier() const;
 
            virtual const CResult SetSharedRandomSeed(const std::pair<CComputationalGraph<T>*, CComputationalGraph<T>*>& prSharedRandomSeed);
            virtual const CResult SetSharedRandomSeed(const std::pair<CComputationalGraph<T>*, CComputationalGraph<T>*>* pPrSharedRandomSeed);          
            virtual const CResult SetSharedRandomSeed(const std::vector<std::pair<CComputationalGraph<T>*, CComputationalGraph<T>*>>& vctSharedRandomSeed);
            virtual const CResult SetSharedRandomSeed(const std::vector<std::pair<CComputationalGraph<T>*, CComputationalGraph<T>*>>* pVctSharedRandomSeed);
            virtual const std::vector<std::pair<CComputationalGraph<T>*, CComputationalGraph<T>*>>& GetSharedRandomSeed(int64_t i64DeviceIndex) const;
 
            virtual const CResult SetMemoryLimitRatio(double f64MemoryLimitRatio = 1.);
            virtual double GetMemoryLimitRatio() const;
 
        protected:
            virtual void Throw(const CResult& res, const wchar_t* pWcsExtraMessage = nullptr) const override;
            virtual const CResult TerminateMiniBatch();
            virtual const CResult InitializeFit();
            virtual const CResult TerminateFit();
            virtual int64_t GetRemainingIteration() const;
            virtual const CResult TerminateCurrentIteration();
            virtual int64_t GetActualMiniBatchSize() const;
            virtual const CResult CopyBatchParams(bool bDifferent);
            virtual const CResult ConvertInternalOptimizerSpecToOptimizer();
            virtual const CResult SynchronizeValueAttribute();
            virtual const CResult UpdateValidationHistory();
 
        protected:
            std::vector<int32_t>& m_vctGPUIndices;
            std::vector<int32_t>& m_vctDeviceIndices;
            bool m_bGPUDevice;
            double m_f64MemoryLimitRatio;
 
            CInternalOptimizerSpecBase<T>* m_pInternalOptimizerSpec;
            std::vector<CInternalOptimizerBase<T>*>& m_vctOptimizer;
            std::vector<CValidatorBase<T>*>& m_vctValidator;
            CInternalOptimizerBase<T>* m_pObBaseOptimizer;
 
            CComputationalGraph<T>* m_pCgFunction;
    
            CLearningRateSchedulerBase<T>* m_pLrScheduler;
            CClassEqualizerBase<T>* m_pClassEqualizer;
            bool m_bClassEqualization;
            CValidatorBase<T>* m_pValidator;
            ELearningRateSchedulerMode m_eLrsMode;
 
            std::vector<int64_t>& m_vctLearnOrders;
            std::vector<int64_t>& m_vctValidationOrders;
            int64_t m_i64CurrentIteration;
            int64_t m_i64TotalIteration;
            int64_t m_i64TotalIterationPerGPU;
            int64_t m_i64Subdivision;
            int64_t m_i64SubMiniBatchSize;
            int64_t m_i64ActualMiniBatchSize;
            int64_t m_i64TotalBatchSize;
            int64_t m_i64ValidationTotalIteration;
            int64_t m_i64CurrentEpoch;
 
            CTensor<T> m_tsrResult;
            int64_t m_i64ResultCount;
 
            T m_tLearningRate;
 
            bool m_bInitialized;
            bool m_bResetDerivatives;
 
            bool m_bDifferent;
            int64_t m_i64PrevInputSize;
            int64_t m_i64PrevPlaceholderSize;
            std::vector<std::tuple<ENodeType, EDataType, EValueAttribute, ENodeOperator, std::vector<int64_t>>>& m_vctPrevInputInfo;
            std::vector<std::vector<int64_t>>& m_vctPrevPlaceholderShape;
 
            int32_t m_i32DeviceCount;
            std::vector<T>& m_vctLearnBatchWeights;
            std::vector<T>& m_vctValidationBatchWeights;
            std::vector<T>& m_vctValidationHistory;
 
            int64_t m_i64SubdivisionMultiplier;
 
            std::vector<std::vector<std::pair<CComputationalGraph<T>*, CComputationalGraph<T>*>>>& m_vct2SharedRandomSeed;
        private:
            template<typename T>
            friend class CValidatorBase;
 
            template<typename T>
            friend class CValidatorForClassifier;
 
            template<typename T>
            friend class CValidatorForSemanticSegmentation;
        };
    }
}
 
#endif