#!/usr/bin/env python
# -*- coding:utf-8 -*-
from typing import Dict, List, Any
import numpy as np
from pandas import DataFrame, Series
import pandas as pd
from statsmodels.formula.api import ols
import statsmodels.api as sm
from collections import ChainMap
from models.AnalysisMongoModel import *
from common.util.my_4_quartiles import my_4_quartiles, myNQuartiles
from patsy.highlevel import dmatrices
import math
import copy
from scipy import *
import collections  # 词频统计库
import jieba  # 结巴分词
# import paddle
import nltk
from nltk.corpus import stopwords
from scipy import stats
import re
from pypinyin import pinyin, lazy_pinyin
from asteval import Interpreter


def mongo_template_logic(PlatID, mongodb_cfg, ProjID, TempID):
    retTemp = mongoTemplateModel(mongodb_cfg)
    DataTemplate = retTemp.SelectTemplate(PlatID, ProjID, TempID)
    #retTemp.close()
    return DataTemplate


def mongo_template_back(PlatID, mongodb_cfg, ProjID, TempID):
    retTemp = mongoTemplateBackModel(mongodb_cfg)
    DataTemplate = retTemp.SelectTemplateBack(PlatID, ProjID, TempID)
    #retTemp.close()
    return DataTemplate


def mongo_template_back_delete(PlatID, mongodb_cfg, ProjID, TempID):
    retTemp = mongoTemplateBackModel(mongodb_cfg)
    retTemp.delete(PlatID, ProjID, TempID)
    #retTemp.close()


def mongo_template_back_insert(mongodb_cfg, templateData):
    retTemp = mongoTemplateBackModel(mongodb_cfg)
    retTemp.insert(templateData)
    #retTemp.close()


def mongo_analysis_template_select(PlatID, mongodb_cfg, ProjID, TempID):
    retAnal = mongoAnalysisModel(mongodb_cfg)
    DataAnalysis = retAnal.SelectAnalysis(PlatID, ProjID, TempID)
    #retAnal.close()
    return DataAnalysis


def mongo_analysis_select(PlatID, mongodb_cfg, ProjID, TempID, RepID):
    retAnal = mongoAnalysisModel(mongodb_cfg)
    DataAnalysis = retAnal.SelectAnalysisReport(PlatID, ProjID, TempID, RepID)
    #retAnal.close()
    return DataAnalysis


def mongo_analysis_delete(PlatID, mongodb_cfg, ProjID, TempID):
    retAnal = mongoAnalysisModel(mongodb_cfg)
    retAnal.delete(PlatID, ProjID, TempID)
    #retAnal.close()


def mongo_analysis_report_delete(PlatID, mongodb_cfg, ProjID, TempID, reportId):
    retAnal = mongoAnalysisModel(mongodb_cfg)
    retAnal.deleteReport(PlatID, ProjID, TempID, reportId)
    #retAnal.close()


def mongo_analysis_insert(mongodb_cfg, analysisData):
    try:
        retAnal = mongoAnalysisModel(mongodb_cfg)
        retAnal.insert(analysisData)
        #retAnal.close()
    except Exception as ex:
        pass


def mongo_template_file_select(PlatID, mongodb_cfg, ProjID, TempID):
    retTemp = mongoTemplateFileModel(mongodb_cfg)
    templateFile = retTemp.SelectTemplateFile(PlatID, ProjID, TempID)
    #retTemp.close()
    return templateFile


def mongo_template_file_update(mongodb_cfg, TempID, templateFilesData):
    retTemp = mongoTemplateFileModel(mongodb_cfg)
    retTemp.update(TempID, templateFilesData)
    #retTemp.close()


def mongo_data_case(PlatID, mongodb_cfg, ProjID, dataSetId):
    retTemp = mongoTemplateModel(mongodb_cfg)
    DataCase = retTemp.SelectData(PlatID, ProjID, dataSetId)
    #retTemp.close()
    return DataCase


def mongo_condition_logic(PlatID, mongodb_cfg, ProjID, dataSetId, VariableID, CasesCondition):
    retVersion = dataversion(mongodb_cfg)
    dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
    #retVersion.close()
    versionNum = None
    for rowItem in dataVersion:
        if rowItem['version'] is not None:
            versionNum = float(rowItem['version'])
    retFrequency = mongoFrequencyModel(mongodb_cfg)
    DataCount = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID, "", CasesCondition)
    #retFrequency.close()
    return DataCount


def mongo_dataSetId_logic(PlatID, mongodb_cfg, ProjID, dataSetId, dataSetIdFlag):
    mongo_dataSetIdFlag = dataSetIdFlag
    versionNum = None
    retVersion = dataversion(mongodb_cfg)
    dataVersion, dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
    #retVersion.close()
    for rowItem in dataVersion:
        if rowItem['version'] is not None:
            versionNum = float(rowItem['version'])
    mongo_dataSetIdFlag['dataSetId'][dataSetId] = dataSetFlag
    mongo_dataSetIdFlag['version'][dataSetId] = versionNum
    return mongo_dataSetIdFlag


def mongo_frequency_logic(PlatID, mongodb_cfg, ProjID, dataSetId, VariableID, phrase, weight, point, idText, deleteRow,
                          dataSetIdFlag, CasesCondition=None):
    """
    :param idText:
    :type point: str
    :param PlatID, mongodb_cfg:
    :param ProjID:
    :param dataSetId:
    :param VariableID:
    :param CasesCondition:
    :return:
    """

    versionNum, variableVarMissing, variableVarValues, varLabel, variableMore, varValues = None, None, None, "", [], ""
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
    else:
        retVersion=dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
    if str(VariableID) == "DWMCtop500":
        testFlag = True
    if str(VariableID).find(",") < 0:
        retMeta = select_meta_variable(mongodb_cfg)
        DataVariable = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, VariableID)
        #retMeta.close()
        for rowItem in DataVariable:
            if 'varValues' in rowItem: variableVarValues=rowItem['varValues']
            if 'varLabel' in rowItem: varLabel = rowItem['varLabel']
            if 'varMissing' in DataVariable: variableVarMissing=rowItem['varMissing']
    else:
        variableMore = str(VariableID).split(',')
    try:
        retFrequency = mongoFrequencyModel(mongodb_cfg)
        DataCount = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID, weight,
                                            CasesCondition)
        #retFrequency.close()
        totalN,validN,invalidN = 0,0,0  # type: float
        jsonValue=dict(head=[],detailValid=[],detailInvalid=[])  # type: Dict[Any, List[Any]]
        #head计算
        validP,invalidP,missingP,DataSource,index,indexText,sortText = 0.0,0.0,0.0,[],0,"",[]
        if idText != "":index = 999999000
        for dataItem in DataCount:
            DataSource.append(dataItem)
        if len(variableMore) > 0 or not variableVarValues or str(variableVarValues).strip() == "" or \
                not check_json_format(str(variableVarValues)):
            for j in DataSource:
                totalN += float(j['count'])
                validFlag = False
                if len(variableMore) > 0:
                    if j['_id'] is not None:
                        validFlag = True
                        for variableItem in variableMore:
                            if  str(j['_id'].get(variableItem)).strip() == 'None' or \
                                    str(j['_id'].get(variableItem)).strip() == 'null' or \
                                    str(j['_id'].get(variableItem)).strip() == '':
                                validFlag = False
                else:
                    if j['_id'] is not None and str(j['_id']).strip() != '' and str(j['_id']).strip() != 'None' and \
                            str(j['_id']).strip() != 'null':
                        validFlag = True
                if validFlag:
                    validN += float(j['count'])
            for j in DataSource:
                if len(variableMore) > 0:
                    validFlag = False
                    if j['_id'] is not None:
                        validFlag = True
                        for variableItem in variableMore:
                            if  j['_id'].get(variableItem) is None or str(j['_id'].get(variableItem)).strip() == '' or \
                                    str(j['_id'].get(variableItem)).strip() == 'None' or \
                                    str(j['_id'].get(variableItem)).strip() == 'null':
                                validFlag = False
                    if validFlag:
                        num, per, perV, text = float(j['count']), 0.0, 0.0, []
                        index += 1
                        if validN != 0:
                            perV = float(j['count']) / float(validN)
                        if totalN != 0:
                            per = float(j['count']) / float(totalN)
                        for variableItem in variableMore:
                            text.append(str(j['_id'].get(variableItem)).strip())
                        jsonValue["detailValid"].append(
                            dict(value=index, text=text, count=int(num), validpercent=perV, percent=per))
                else:
                    if j['_id'] is not None and str(j['_id']).strip() != '' and str(j['_id']).strip() != 'None' and \
                            str(j['_id']).strip() != 'null':
                        num, per, perV = float(j['count']), 0.0, 0.0
                        index += 1
                        if validN != 0:
                            perV = float(j['count']) / float(validN)
                        if totalN != 0:
                            per = float(j['count']) / float(totalN)
                        if is_number_int(str(j['_id']).strip()):
                            if idText == "": sortText.append(str(j['_id']).strip())
                            jsonValue["detailValid"].append(
                                dict(value=int(float(str(j['_id']).strip())), text=str(j['_id']).strip(), count=int(num),
                                     validpercent=perV, percent=per))
                        else:
                            valueText = str(j['_id']).strip()
                            if idText == "": sortText.append(valueText)
                            if idText != "" and "".join(list(filter(str.isdigit, str(j['_id']).strip()))) != "":
                                indexText = "".join(list(filter(str.isdigit, str(j['_id']).strip())))
                                if indexText == str(j['_id']).strip():
                                    valueText = str(j['_id']).strip()
                                else:
                                    valueText = str(j['_id']).strip().replace(indexText, '')
                                if str(j['_id']).strip().find(indexText) < 0:
                                    charEnd = -1
                                    if str(j['_id']).strip().find(indexText[0:1]) == 0:
                                        for charIndex in range(len(indexText)):
                                            if str(j['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                                charEnd = charIndex
                                                break
                                    if charEnd != -1:
                                        indexText = indexText[0:charEnd]
                                        valueText = str(j['_id']).strip()[charEnd:len(str(j['_id']).strip())]
                                    else:
                                        indexText = ''
                                        valueText = str(j['_id']).strip()
                                if indexText == "" : indexText = str(index)
                                index = int(float(indexText))
                            jsonValue["detailValid"].append(dict(value=index, text=valueText, count=int(num),
                                                                 validpercent=perV, percent=per))
            if float(totalN) != 0:
                if isinstance(phrase, str) and str(phrase).strip() != "":
                    varLabel = phrase
                validP = float(validN) / float(totalN)
                missingP = float(totalN - validN) / float(totalN)
                jsonValue["detailValid"].sort(key=lambda x: x["validpercent"], reverse=True)
            jsonValue["head"].append(
                dict(varLabel=varLabel, variableID=VariableID, count=int(validN), totalN=int(totalN), validN=int(validN),
                     validP=validP,missingN=int(totalN - validN), missingP=missingP, length=len(jsonValue["detailInvalid"]),
                     varValues=variableVarValues,analysis="frequency"))
        else:
            varValues = variableVarValues
            for j in DataSource:
                totalN+=float(j['count'])
                for key,value in json.loads(variableVarValues).items():
                    if str(j['_id']) == str(key) :
                        validN += float(j['count'])
                        break
                if variableVarMissing :
                    for key,value in json.loads(variableVarMissing).items():
                        if j['_id'] == str(key) :
                            invalidN += float(j['count'])
                            break
            for j in DataSource:
                for key,value in json.loads(variableVarValues).items():
                    if j['_id'] == str(key) :
                        num=float(j['count'])
                        per,perV = 0.0,0.0
                        if validN!=0 :
                            perV=float(j['count'])/float(validN)
                        if totalN!=0 :
                            per=float(j['count'])/float(totalN)
                        jsonValue["detailValid"].append(
                            dict(value=int(float(j['_id'])),text=str(value),count=int(num),validpercent=perV,percent=per))
                        break
                if variableVarMissing :
                    for key,value in json.loads(variableVarMissing).items() :
                        if j['_id'] == str(key) :
                            num=float(j['count'])
                            per,perV = 0.0,0.0
                            if validN!=0 :
                                perV=float(j['count'])/float(invalidN)
                            if totalN!=0 :
                                per=float(j['count'])/float(totalN)
                            jsonValue["detailInvalid"].append(
                                dict(value=int(float(j['_id'])),text=str(value),count=int(num),validpercent=perV,percent=per))
                            break
            if float(totalN) != 0:
                validP = float(validN) / float(totalN)
                invalidP = float(invalidN) / float(totalN)
                missingP = float(totalN - validN - invalidN) / float(totalN)
                if isinstance(phrase, str) and str(phrase).strip() != "":
                    varLabel = phrase
            jsonValue["head"].append(
                dict(varLabel=varLabel, variableID=VariableID, count=int(validN) ,totalN=int(totalN), validN=int(validN),
                     validP=validP,invalidN=int(invalidN), invalidP=invalidP, missingN=int(totalN - validN - invalidN),
                     missingP=missingP, length=len(jsonValue["detailInvalid"]), varValues=varValues,
                     analysis="frequency"))
            if totalN - validN - invalidN > 0:
                num = totalN - validN - invalidN
                perV,per = 1.0,0.0
                if totalN != 0:
                    per = (totalN - validN - invalidN) / totalN
                jsonValue["detailInvalid"].append(dict(value="", text="", count=int(num), validpercent=perV,
                                                       percent=per))
        if point != "":
            jsonValue["detailValid"] = formatHundredPercent(jsonValue["detailValid"], point)
        if len(sortText) > 0:
            sortTextArray = sorted(sortText, key=lazy_pinyin)
            for detailValidIndex in range(len(jsonValue["detailValid"])):
                jsonValue["detailValid"][detailValidIndex]["value"] = \
                    sortTextArray.index(jsonValue["detailValid"][detailValidIndex]["text"]) + 1
        jsonValue["detailValid"].sort(key=lambda x: x["value"], reverse=False)
        if str(deleteRow) != "" and (isinstance(deleteRow, dict) or
                                     (isinstance(deleteRow, str) and check_json_format(deleteRow))):
            if isinstance(deleteRow, str): deleteRow = json.loads(deleteRow)
            detailRemove = []
            if 'length' in deleteRow.keys():
                jsonValue["detailValid"].sort(key=lambda x: x["count"], reverse=True)
                if len(jsonValue["detailValid"]) > deleteRow['length']:
                    for rowIndex in range(deleteRow['length'], len(jsonValue["detailValid"])):
                        if jsonValue["detailValid"][deleteRow['length'] - 1]['count'] != \
                                jsonValue["detailValid"][rowIndex]['count']:
                            detailRemove.append(jsonValue["detailValid"][rowIndex])
                if len(detailRemove) > 1:
                    if 'otherName' in deleteRow.keys():
                        num, perV, per = 0, 0.0, 0.0
                        for index, detailItem in enumerate(detailRemove):
                            num += detailItem['count']
                            perV += detailItem['validpercent']
                            per += detailItem['percent']
                        jsonValue["detailValid"].append(dict(value=999999999, text=deleteRow['otherName'],
                                                               count=int(num), validpercent=perV,
                                                               percent=per))
                else:
                    detailRemove = []
            else:
                for index, detailItem in enumerate(jsonValue["detailValid"]):
                    removeFlag = False
                    for key, value in deleteRow.items():
                        if detailItem[key] < value:
                            removeFlag = True
                    if removeFlag:
                        detailRemove.append(detailItem)
            for index, detailItem in enumerate(detailRemove):
                jsonValue["detailValid"].remove(detailItem)
        return jsonValue
    except Exception as ex:
        return {}


def mongo_means_logic(PlatID, mongodb_cfg, ProjID, dataSetId, VariableID, VariableIDgroup, optionIDgroup,
                      IntervalRange, phrase, fullMarks, denominator, weight, formula, dataSetIdFlag, noCondition,
                      replaceText, CasesCondition=None):
    """
    :type PlatID, mongodb_cfg: str
    :param PlatID, mongodb_cfg:
    :type ProjID: str
    :param ProjID: object
    :param dataSetId: object
    :param VariableID: object
    :param CasesCondition: object
    :return:
    """
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID,dataSetId)
        #retVersion.close()
    versionNum = None
    try:
        varLabel, jsonValue = "", dict(head=[], detailValid=[])
        if VariableID != "":
            retMeta = select_meta_variable(mongodb_cfg)
            DataVariable = retMeta.SelectMetaVariable(PlatID, ProjID,dataSetId,versionNum,VariableID)
            #retMeta.close()
            for rowItem in DataVariable :
                if 'varLabel' in rowItem: varLabel = rowItem['varLabel']
        retMeans = mongoMeansModel(mongodb_cfg)
        DataSum = retMeans.SumMongo(dataSetFlag, ProjID, dataSetId,versionNum,VariableID,VariableIDgroup,"",weight,
                                    CasesCondition)
        #retMeans.close()
        DataSort, DataWeight, replaceFrom, replaceTo = [], [], [], []
        if replaceText is None or not isinstance(replaceText, list):
            replaceText = []
        for i in range(len(replaceText)):
            if isinstance(replaceText[i][0], float):
                replaceFrom.append(replaceText[i][0])
            else:
                replaceFrom.append(float(str(replaceText[i][0])))
            if isinstance(replaceText[i][1], float):
                replaceTo.append(replaceText[i][1])
            else:
                if is_number(str(replaceText[i][1])):
                    replaceTo.append(float(str(replaceText[i][1])))
                else:
                    replaceTo.append('')
        if str(VariableID).strip() != "":
            for j in DataSum:
                if VariableID in j.keys() and str(j[VariableID]).strip() != '' and is_number(str(j[VariableID]).strip()):
                    if weight != "" and str(j[weight]).strip() != '' and is_number(str(j[weight]).strip()):
                        DataWeight.append(float(str(j[weight]).strip()))
                    if float(str(j[VariableID]).strip()) in replaceFrom:
                        if isinstance(replaceTo[replaceFrom.index(float(str(j[VariableID]).strip()))], float):
                            DataSort.append(replaceTo[replaceFrom.index(float(str(j[VariableID]).strip()))])
                    else:
                        DataSort.append(float(str(j[VariableID]).strip()))
            if isinstance(IntervalRange, list):
                DataSortCopy = DataSort[:]
                for num in DataSort:
                    if len(IntervalRange) == 2:
                        if num < float(IntervalRange[0]) or num > float(IntervalRange[1]):
                            DataSortCopy.remove(num)
                    elif len(IntervalRange) == 3:
                        if num <= float(IntervalRange[0]) or num > float(IntervalRange[1]):
                            DataSortCopy.remove(num)
                    else:
                        if IntervalRange[3] == 1:
                            if num <= float(IntervalRange[0]) or num >= float(IntervalRange[1]):
                                DataSortCopy.remove(num)
                        else:
                            if num < float(IntervalRange[0]) or num >= float(IntervalRange[1]):
                                DataSortCopy.remove(num)
                DataSort = DataSortCopy
            if noCondition == "":
                for numIndex, numItem in enumerate(DataSort):
                    jsonValue["detailValid"].append(
                        dict(value=numIndex, text=str(numIndex), count=1, data=numItem))
        else:
            for dataItem in DataSum:
                num = 0.0
                if isinstance(optionIDgroup, str) and optionIDgroup == "":
                    for variableItem in VariableIDgroup:
                        if variableItem in dataItem.keys() and dataItem[variableItem] is not None and \
                                str(dataItem[variableItem]).strip() != "":
                            num += float(dataItem[variableItem])
                else:
                    fullMarksNum = 0
                    for optionItem in optionIDgroup:
                        fullMarksNum += 1
                        if 0 < int(optionItem) <= 150:
                            if VariableIDgroup[0] in dataItem.keys() and dataItem[VariableIDgroup[0]] is not None and \
                                    str(dataItem[VariableIDgroup[0]]).strip() != "":
                                num += float(str(dataItem[VariableIDgroup[0]])[int(optionItem) - 1:int(optionItem)])
                        else:
                            if VariableIDgroup[1] in dataItem.keys() and dataItem[VariableIDgroup[1]] is not None and \
                                    str(dataItem[VariableIDgroup[1]]).strip() != "":
                                num += float(str(dataItem[VariableIDgroup[1]])[int(optionItem) - 151:int(optionItem) - 150])
                    fullMarks = str(fullMarksNum)
                if num in replaceFrom:
                    if isinstance(replaceTo[replaceFrom.index(num)], float):
                        num = replaceTo[replaceFrom.index(num)]
                if isinstance(IntervalRange, list):
                    if len(IntervalRange) == 2:
                        if  float(IntervalRange[0]) <= num <= float(IntervalRange[1]):
                            DataSort.append(num)
                    elif len(IntervalRange) == 3:
                        if  float(IntervalRange[0]) < num <= float(IntervalRange[1]):
                            DataSort.append(num)
                    else:
                        if IntervalRange[3] == 1:
                            if float(IntervalRange[0]) < num < float(IntervalRange[1]):
                                DataSort.append(num)
                        else:
                            if float(IntervalRange[0]) <= num < float(IntervalRange[1]):
                                DataSort.append(num)
                else:
                    DataSort.append(num)
        sumData,cnt,avg,maxData,mid,minData,mode,std,var,masteryRate,passRate = 0,0,0,0,0,0,0,0,0,0.0,0.0
        Percent5,Percent10,Percent25,Percent75,Percent90,Percent95 = 0,0,0,0,0,0
        if len(DataSort) > 0:
            cnt = len(DataSort)
            maxData=np.max(DataSort)
            sumData=np.sum(DataSort)
            minData=np.min(DataSort)
            if len(DataWeight) > 0:
                avg = np.average(DataSort, 0, DataWeight)
                PercentArray = weighted_quantile(DataSort, [0.05, 0.1, 0.25, 0.75, 0.9, 0.95], DataWeight)
                Percent5, Percent10, Percent25, Percent75, Percent90, Percent95 = \
                    PercentArray[0], PercentArray[1], PercentArray[2], PercentArray[3], PercentArray[4], PercentArray[5]
                std = np.sqrt(np.cov(DataSort, aweights=DataWeight))
            else:
                avg = np.average(DataSort)
                std = np.std(DataSort, axis=0, ddof=1)                  # 数据只有1条会报错，除0错误，返回值nan
                Percent5 = np.percentile(DataSort, 5, method="weibull")
                Percent10 = np.percentile(DataSort, 10, method="weibull")
                Percent25 = np.percentile(DataSort, 25, method="weibull")
                Percent75 = np.percentile(DataSort, 75, method="weibull")
                Percent90 = np.percentile(DataSort, 90, method="weibull")
                Percent95 = np.percentile(DataSort, 95, method="weibull")
            if np.isnan(std):std = 0.0
            var = np.var(DataSort)
            mid = np.median(DataSort)
            gm = pd.Series(data=DataSort)
            mode = gm.mode()[0]
            if isinstance(phrase, str) and str(phrase).strip() != "":
                varLabel = phrase
        if fullMarks != "" and float(fullMarks) != 0:
            masteryRate = avg / float(fullMarks)
        if denominator != "" and float(denominator) != 0:
            passRate = float(cnt) / float(denominator)
        jsonValue["head"].append(
            dict(varLabel=varLabel, variableID=VariableID, sum=sumData, count=cnt, average=avg, passRate=passRate,
                 max=maxData, median=mid,min=minData, mode=mode, stdev=std, variance=var, masteryRate=masteryRate,
                 Percent5=Percent5, Percent10=Percent10,Percent25=Percent25,Percent75=Percent75,
                 Percent90=Percent90, Percent95=Percent95,analysis="means"))
        if formula != '': calculation_formula(jsonValue["head"], formula)
        return jsonValue
    except Exception as ex:
        return {}


def weighted_quantile(values, quantiles, sample_weight=None, values_sorted=False, old_style=False):
    """ Very close to numpy.percentile, but supports weights.
    NOTE: quantiles should be in [0, 1]!
    :param values: numpy.array with data
    :param quantiles: array-like with many quantiles needed
    :param sample_weight: array-like of the same length as `array`
    :param values_sorted: bool, if True, then will avoid sorting of
        initial array
    :param old_style: if True, will correct output to be consistent
        with numpy.percentile.
    :return: numpy.array with computed quantiles.
    """
    values = np.array(values)
    quantiles = np.array(quantiles)
    if sample_weight is None:
        sample_weight = np.ones(len(values))
    sample_weight = np.array(sample_weight)
    assert np.all(quantiles >= 0) and np.all(quantiles <= 1), \
        'quantiles should be in [0, 1]'

    if not values_sorted:
        sorter = np.argsort(values)
        values = values[sorter]
        sample_weight = sample_weight[sorter]

    weighted_quantiles = np.cumsum(sample_weight) - 0.5 * sample_weight
    if old_style:
        # To be convenient with numpy.percentile
        weighted_quantiles -= weighted_quantiles[0]
        weighted_quantiles /= weighted_quantiles[-1]
    else:
        weighted_quantiles /= np.sum(sample_weight)
    return np.interp(quantiles, weighted_quantiles, values)


def weighted_percentile(values, percents, weights=None):
    ''' percents in units of 1%
        weights specifies the frequency (count) of data.
    '''
    if weights is None:
        return np.percentile(values, percents)
    ind = np.argsort(values)
    d = values[ind]
    w = weights[ind]
    p = 1.*w.cumsum()/w.sum()*100
    return np.interp(percents, p, d)


def mongo_means2_logic(PlatID, mongodb_cfg, ProjID, dataSetId, VariableIDgroup, optionIDgroup,
                       phrase, fullMarks, denominator, weight, formula, dataSetIdFlag, CasesCondition=None):
    """
    :type PlatID, mongodb_cfg: str
    :param PlatID, mongodb_cfg:
    :type ProjID: str
    :param ProjID: object
    :param dataSetId: object
    :param VariableIDgroup: object
    :param CasesCondition: object
    :return:
    """

    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID,dataSetId)
        #retVersion.close()
    versionNum = None
    try:
        varLabel, jsonValue, VariableID = "", dict(head=[], detailValid=[]), VariableIDgroup[0]
        if len(VariableIDgroup) == 1:
            retMeta = select_meta_variable(mongodb_cfg)
            DataVariable = retMeta.SelectMetaVariable(PlatID,ProjID,dataSetId,versionNum,VariableIDgroup[0])
            #retMeta.close()
            for rowItem in DataVariable :
                if 'varLabel' in rowItem: varLabel = rowItem['varLabel']
        if weight != "":VariableIDgroup[0] = VariableIDgroup[0] + "*" + weight
        retMeans = mongoMeansModel(mongodb_cfg)
        try:
            DataSum = retMeans.SumMongoAvg(dataSetFlag, ProjID, dataSetId, versionNum, VariableIDgroup, optionIDgroup,
                                           "", weight, CasesCondition)
        except Exception as ex:
            DataSum = []
        #retMeans.close()
        sumData,cnt,avg,maxData,minData,masteryRate,passRate,VariableMore,VariableItem = 0,0,0,0,0,0.0,0.0,[],""
        if VariableIDgroup[0].find("*") > -1:
            VariableMore = VariableIDgroup[0].split('*')
        elif VariableIDgroup[0].find("/") > -1:
            VariableMore = VariableIDgroup[0].split('/')
        elif VariableIDgroup[0].find("+") > -1:
            VariableMore = VariableIDgroup[0].split('+')
        elif VariableIDgroup[0].find("-") > -1:
            VariableMore = VariableIDgroup[0].split('-')
        if len(VariableMore) > 0:
            VariableItem = VariableMore[0]
        else:
            VariableItem = VariableIDgroup[0]
        for dataItem in DataSum:
            if dataItem[VariableItem + 'avg'] is not None and \
                    str(dataItem[VariableItem + 'avg']).strip() != '':
                if weight != "":
                    avg = float(str(dataItem[VariableItem + 'sum']).strip()) / float(str(dataItem['count']).strip())
                else:
                    avg = float(str(dataItem[VariableItem + 'avg']).strip())
                sumData = float(str(dataItem[VariableItem + 'sum']).strip())
                maxData = float(str(dataItem[VariableItem + 'max']).strip())
                minData = float(str(dataItem[VariableItem + 'min']).strip())
                if dataItem['count'] is not None and str(dataItem['count']).strip() != '':
                    cnt = float(str(dataItem['count']).strip())
        if isinstance(phrase, str) and str(phrase).strip() != "":
            varLabel = phrase
        if fullMarks != "" and float(fullMarks) != 0:
            masteryRate = avg / float(fullMarks)
        if denominator != "" and float(denominator) != 0:
            passRate = float(cnt) / float(denominator)
        jsonValue["head"].append(
            dict(varLabel=varLabel, variableID=VariableID, sum=sumData, count=cnt, average=avg, passRate=passRate,
                 max=maxData, min=minData, masteryRate=masteryRate,mode=0, stdev=0, variance=0,Percent5=0, Percent10=0,
                 Percent25=0, Percent75=0,Percent90=0, Percent95=0, median=0,analysis="means"))
        if formula != '': calculation_formula(jsonValue["head"], formula)
        return jsonValue
    except Exception as ex:
        return {}


def mongo_multiple_logic(PlatID, mongodb_cfg, ProjID, dataSetId, VariableID, phrase, weight, dataSetIdFlag,
                         CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion=dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID,dataSetId)
        #retVersion.close()
    versionNum = None
    retMeta = select_meta_variable(mongodb_cfg)
    DataVariable = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, VariableID)
    #retMeta.close()
    try:
        retFrequency = mongoFrequencyModel(mongodb_cfg)
        DataCount = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID, weight,
                                            CasesCondition)
        #retFrequency.close()
        variableVarMissing, variableVarValues, varLabel, numFlag = None, None, "", True
        for rowItem in DataVariable:
            if 'varValues' in rowItem: variableVarValues = rowItem['varValues']
            if 'varLabel' in rowItem: varLabel = rowItem['varLabel']
            if 'varMissing' in DataVariable: variableVarMissing = rowItem['varMissing']
        if not variableVarValues or str(variableVarValues).strip()  == "" or not \
                check_json_format(str(variableVarValues)):
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            DataCountVar = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID, weight,
                                                   CasesCondition)
            #retFrequency.close()
            keys, keys2, values, index = [], [], [], 0
            for j in DataCountVar:
                if j['_id'] is not None and str(j['_id']).strip() != "" and str(j['_id']).strip() != "None":
                    if "," in j['_id']:
                        varstr = str(j['_id']).strip().split(",")
                    elif "." in j['_id']:
                        varstr = str(j['_id']).strip().split(".")
                    else:
                        varstr = [str(j['_id']).strip()]
                    for itemVar in varstr:
                        if itemVar not in values:
                            if not is_number(itemVar): numFlag = False
                            index += 1
                            keys.append(str(index))
                            keys2.append(itemVar)
                            values.append(itemVar)
            if not numFlag:
                variableVarValues = json.dumps(dict(zip(keys, values)))
            else:
                variableVarValues = json.dumps(dict(zip(keys2, values)))
        variableVarValues = json.loads(variableVarValues)
        if isinstance(phrase, dict) and numFlag:
            for key, value in variableVarValues.items():
                for k, v in phrase.items():
                    if key == k:
                        variableVarValues[key] = v
                        break
        caseN,validN,missingN,invalidN,valueCount,missingCount,delimiter = 0.0,0.0,0.0,0.0,{},{},"."
        for key, value in variableVarValues.items():
            valueCount[key] = 0
        if variableVarMissing:
            for key, value in json.loads(variableVarMissing).items():
                missingCount[key] = 0
        for j in DataCount:
            if not j['_id'] or str(j['_id']).strip() == ""  or str(j['_id']).strip() == "None":
                missingN += float(j["count"])
            else:
                if "," in j['_id']:
                    delimiter = ","
                for key, value in variableVarValues.items():
                    if (numFlag and (delimiter + str(j['_id']) + delimiter).find(delimiter + str(key) + delimiter) >= 0)\
                            or (not numFlag and
                             (delimiter + str(j['_id']) + delimiter).find(delimiter + str(value) + delimiter) >= 0) :
                        for k, v in valueCount.items():
                            if key == k:
                                valueCount[k] = v + float(j["count"])
                                validN += float(j["count"])
                if variableVarMissing:
                    for key,value in json.loads(variableVarMissing).items():
                        if (delimiter + str(j['_id']) + delimiter).find(delimiter + str(key) + delimiter) >= 0:
                            for k, v in missingCount.items():
                                if key == k:
                                    missingCount[k] = v + float(j["count"])
                                    invalidN += float(j["count"])
                caseN += float(j["count"])
        jsonValue = dict(head=[], detailValid=[], detailInvalid=[])
        if isinstance(phrase, str) and str(phrase).strip() != "":
            varLabel = phrase
        for key,value in variableVarValues.items():
            for k,v in valueCount.items():
                if str(key)==str(k):
                    num,caseP,validP = float(v),0.0,0.0
                    if num > 0:
                        if caseN != 0:
                            caseP = float(v) / caseN
                        if validN != 0:
                            validP = float(v) / validN
                        jsonValue["detailValid"].append(dict(value=int(float(key)), text=value, count=int(num),
                                                             casepercent=caseP, validpercent=validP))
        if variableVarMissing :
            for key,value in json.loads(variableVarMissing).items():
                for k,v in missingCount.items():
                    if key==k:
                        num,caseP,validP = float(v),0.0,0.0
                        if num > 0:
                            if caseN!=0 :
                                caseP=float(v)/invalidN
                            if validN!=0 :
                                validP=float(v)/validN
                            jsonValue["detailInvalid"].append(
                                dict(value=float(key),text=value,count=int(num),casepercent=caseP,percent=validP))
        jsonValue["detailValid"].sort(key=lambda x: x["value"], reverse=False)
        jsonValue["head"].append(
            dict(varLabel=varLabel, variableID=VariableID, count=int(caseN), caseN=int(caseN), validN=int(validN),
                 invalidN=int(invalidN), length=len(jsonValue["detailValid"]), analysis="multiple"))
        return jsonValue
    except Exception as ex:
        return {}


#多重响应二分法
def mongo_multiple2_logic(PlatID, mongodb_cfg, ProjID, dataSetId, VariableID, phrase, OptionID, weight, dataSetIdFlag,
                          CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
    versionNum = None

    try:
        caseN, validN, invalidN, valueCount, varLabel, value, caseListN = 0.0, 0.0, 0.0, {}, {}, {}, {}
        if str(VariableID).find(",") > -1:
            VariableOptionID = str(VariableID).strip().split(',')
            VariableID = VariableOptionID[0]
        else:
            if not isinstance(OptionID, list):
                fromTO = str(OptionID).strip().split('-')
                OptionID = []
                if float(fromTO[0]) == 0:
                    for n in range(int(fromTO[1])):
                        OptionID.append(str(n))
                else:
                    for n in range(int(fromTO[1])):
                        OptionID.append(str(n + 1))
            VariableOptionID = []
            for itemOpt in OptionID:
                VariableOptionID.append(VariableID + str(itemOpt))
        
        retMeta = select_meta_variable(mongodb_cfg)
        variable_metadata = {}
        for itemOpt in VariableOptionID:
            DataVariable = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, str(itemOpt))
            for rowItem in DataVariable:
                if 'varLabel' in rowItem:
                    variable_metadata[itemOpt] = rowItem['varLabel']
                else:
                    variable_metadata[itemOpt] = ""
        
        retFrequency = mongoFrequencyModel(mongodb_cfg)
        
        all_count_data = retFrequency.CountMongoMultiple(dataSetFlag, ProjID, dataSetId, versionNum,
                                                        VariableOptionID, weight, CasesCondition)
        
        for itemInd, itemOpt in enumerate(VariableOptionID):
            valueCount[itemInd] = 0
            caseListN[itemInd] = 0.0
            varLabel[itemInd] = variable_metadata.get(itemOpt, "")
            if isinstance(phrase, list) and len(phrase) > itemInd:
                varLabel[itemInd] = phrase[itemInd]
            if is_number_int(itemOpt):
                value[itemInd] = itemOpt
            else:
                value[itemInd] = str(itemInd + 1)
            
            DataCount = all_count_data.get(itemOpt, [])
            for j in DataCount:
                if str(j['_id']).strip() != '' and \
                        ((is_number(str(j['_id']).strip()) and float(str(j['_id']).strip()) >= 0)
                         or str(j['_id']).strip() == '勾选' or str(j['_id']).strip() == '未勾选'):
                    caseListN[itemInd]  += float(j['count'])
                    if (is_number(str(j['_id']).strip()) and float(str(j['_id']).strip()) > 0) or \
                            str(j['_id']).strip() == '勾选':
                        valueCount[itemInd] += float(j['count'])
                        validN += float(j['count'])
        
        CasesCondition0 = copy.deepcopy(CasesCondition)
        for itemOpt in VariableOptionID:
            if isinstance(CasesCondition0, list):
                CasesCondition0[0].append(str(itemOpt))
                CasesCondition0[1].append("0")
            else:
                CasesCondition0 = [[str(itemOpt)], ["0"]]
        
        DataCount = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum,
                                            str(VariableOptionID[0]), weight, CasesCondition0)
        #retFrequency.close()
        for caseValue in caseListN.values():
            if caseN < caseValue: caseN = caseValue
        for j in DataCount:
            caseN = caseN - float(j['count'])
        jsonValue = dict(head=[], detailValid=[], detailInvalid=[])
        for k,v in valueCount.items():
            num,caseP,validP = float(v),0.0,0.0
            if num > 0:
                if caseN != 0:
                    caseP = float(v) / caseN
                if validN != 0:
                    validP = float(v) / validN
                jsonValue["detailValid"].append(
                    dict(value=int("".join(list(filter(str.isdigit, value[k])))), text=varLabel[k], count=int(num),
                         casepercent=caseP, validpercent=validP))
        jsonValue["detailValid"].sort(key=lambda x: x["value"], reverse=False)
        jsonValue["head"].append(
            dict(varLabel="", variableID=VariableID, count=int(caseN), caseN=int(caseN), validN=int(validN),
                 invalidN=int(invalidN), length=len(jsonValue["detailValid"]), analysis="multiple"))
        return jsonValue
    except Exception as ex:
        return {}


def mongo_wordcloud_logic(PlatID, mongodb_cfg, ProjID, dataSetId, VariableID, phrase, dataSetIdFlag, removeWord,
                          addWord, CasesCondition=None):
    """
    :param PlatID, mongodb_cfg:
    :param ProjID:
    :param dataSetId:
    :param VariableID:
    :param CasesCondition:
    :return:
    """
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion=dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
        #retVersion.close()
    versionNum = None
        #for rowItem in dataVersion:
        #    if rowItem['version'] is not None:
        #        versionNum=float(rowItem['version'])
    retMeta = select_meta_variable(mongodb_cfg)
    DataVariable = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, VariableID)
    #retMeta.close()
    retFrequency = mongoFrequencyModel(mongodb_cfg)
    DataCount = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID, "", CasesCondition)
    #retFrequency.close()
    totalN, validN, validP, missingP, varLabel = 0.0, 0.0, 0.0, 0.0, ""
    for rowItem in DataVariable:
        if 'varLabel' in rowItem: varLabel = rowItem['varLabel']
    jsonValue = dict(head=[],detailValid=[])  # type: Dict[Any, List[Any]]
    #head计算
    DataSource = []
    for dataItem in DataCount:
        DataSource.append(dataItem)
    for j in DataSource:
        totalN += float(j['count'])
        if j['_id'] is not None and str(j['_id']).strip() != '' and str(j['_id']).strip() != 'None' and \
                str(j['_id']).strip() != 'null':
            validN += float(j['count'])
    for j in DataSource:
        if j['_id'] is not None and str(j['_id']).strip() != '' and str(j['_id']).strip() != 'None' and \
                str(j['_id']).strip() != 'null':
            num,per,perV = float(j['count']),0.0,0.0
            if validN != 0 :
                perV = float(j['count'])/float(validN)
            if totalN != 0 :
                per = float(j['count'])/float(totalN)
            jsonValue["detailValid"].append(
                dict(value='0',text=str(j['_id']).strip(),count=int(num),validpercent=perV,percent=per))
    if float(totalN) != 0:
        if isinstance(phrase, str) and str(phrase).strip() != "":
            varLabel = phrase
        validP = float(validN) / float(totalN)
        missingP = float(totalN - validN) / float(totalN)
        jsonValue["detailValid"].sort(key=lambda x: x["validpercent"], reverse=True)
    # 词云分析
    ColumnIDs = VariableID.split(",")
    wordFreObj = mongoCorrelationAnalysisModel(mongodb_cfg)
    wordData = wordFreObj.showDatas(dataSetFlag, ProjID, dataSetId, versionNum, ColumnIDs, CasesCondition)

    wordDataList, seg_list_exact, addWordList = [], [], None
    nltk.data.path.append(r'path_to_nltk_data')  # 替换为nltk数据所在的路径
    remove_words = set(stopwords.words('chinese'))  # 获取中文停用词
    if removeWord is not None:
        if isinstance(removeWord, str):
            removeWordList = removeWord.split('\n')
        else:
            removeWordList = removeWord
        for removeWordItem in removeWordList:
            remove_words.add(removeWordItem)
    if addWord is not None:
        if isinstance(addWord, str):
            addWordList = addWord.split('\n')
        else:
            addWordList = addWord
        for addWordItem in addWordList:
            jieba.add_word(addWordItem)
            if addWordItem in remove_words:
                remove_words.remove(addWordItem)
    for wordDataItem in wordData:
        if ColumnIDs[0] in wordDataItem.keys() and wordDataItem[ColumnIDs[0]] is not None and\
                wordDataItem[ColumnIDs[0]] not in remove_words:
            wordDataList.append(str(wordDataItem[ColumnIDs[0]]))
    string_data = "".join(wordDataList)

    pattern = re.compile('\t|\n|\.|-|:|;|\)|\(|\?|（|）|\|"|\u3000|,')  # 定义正则表达式匹配模式
    string_data = re.sub(pattern, '', string_data)  # 将符合模式的字符去除
    object_list = []
    try:
        '''
        from pyhanlp import HanLP, CustomDictionary
        if addWordList is not None and isinstance(addWordList, list):
            for addWordItem in addWordList:
                CustomDictionary.add(addWordItem)

        seg_list_exact = HanLP.segment(string_data)
        for word in seg_list_exact:  # 循环读出每个分词
            wordText = re.sub(r'\d+', '', word.word)
            if len(wordText) > 1 and wordText not in remove_words and wordText.strip() != '':  # 1<长度 并且 不在去除词库中
                object_list.append(wordText)  # 分词追加到列表
        '''
        import jieba.posseg as pseg
        seg_list_exact = pseg.cut(string_data, use_paddle=True)  # 精确模式分词
        for word, flag in seg_list_exact:  # 循环读出每个分词
            wordText = re.sub(r'\d+', '', word)
            if flag != 'd' and len(
                    wordText) > 1 and wordText not in remove_words and wordText.strip() != '':  # 1<长度 并且 不在去除词库中
                object_list.append(wordText)  # 分词追加到列表
    except Exception as e:
        seg_list_exact = jieba.cut_for_search(string_data)
        for word in seg_list_exact:  # 循环读出每个分词
            wordText = re.sub(r'\d+', '', word)
            if len(wordText) > 1 and wordText not in remove_words and wordText.strip() != '':  # 1<长度 并且 不在去除词库中
                object_list.append(wordText)  # 分词追加到列表

    word_counts = collections.Counter(object_list)  # 对分词做词频统计
    word_counts_top10 = word_counts.most_common(200)  # 获取前200最高频的词

    jsonValue["head"].append(
        dict(varLabel=varLabel, variableID=VariableID, count=int(validN), totalN=int(totalN), validN=int(validN),
             validP=validP, missingN=int(totalN - validN), missingP=missingP, wordCloud=dict(word_counts_top10),
             analysis="wordcloud"))
    return jsonValue


# -----<交叉表：列联表分析>----- #
def mongo_crosstabs_logic(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID, Group, deleteRow, idText, weight,
                          dataSetIdFlag, CasesCondition=None):
    if RowID == 'level2':
        testFlag = True
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion=dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID,ProjID,dataSetId)
        #retVersion.close()
    versionNum = None

    retMeta = select_meta_variable(mongodb_cfg)
    DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum,RowID)
    DataColumnID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId,versionNum, ColumnID)
    #retMeta.close()
    try:
        retCrosstabs = mongoCrosstabsModel(mongodb_cfg)
        DataCount = retCrosstabs.CrosstabsCountMongo(dataSetFlag, ProjID, dataSetId, versionNum,RowID, ColumnID,
                                                     weight, CasesCondition)
        #retCrosstabs.close()
        rowVarLabel, columnVarLabel, rowIDValues, columnIDValues, variableMore, rowVarType, columnVarType = \
            "","",None,None,[], "", ""
        if str(ColumnID).find(",") > -1:variableMore = str(ColumnID).split(',')
        for rowItem in DataRowID :
            if 'varValues' in rowItem: rowIDValues = rowItem['varValues']
            if 'varLabel' in rowItem: rowVarLabel= rowItem['varLabel']
            if 'varType' in rowItem: rowVarType = rowItem['varType']
        for rowItem in DataColumnID :
            if 'varValues' in rowItem: columnIDValues = rowItem['varValues']
            if 'varLabel' in rowItem: columnVarLabel = rowItem['varLabel']
            if 'varType' in rowItem: columnVarType = rowItem['varType']
        if not rowIDValues or str(rowIDValues).strip() == "" or not check_json_format(str(rowIDValues).strip()):
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            RowData = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, "", CasesCondition)
            #retFrequency.close()
            index, rowIDValuesJson1, rowIDValuesJson2, numFlag, indexText, valueText, sortText = \
                0, [], [], True, "", "", []
            if idText != "":index = 999999000
            for dataItem in RowData:
                if dataItem['_id'] is not None and str(dataItem['_id']).strip() != '' and \
                        str(dataItem['_id']).strip() != 'None' and str(dataItem['_id']).strip() != 'null':
                    if idText != "" and "".join(list(filter(str.isdigit, str(dataItem['_id']).strip()))) != "":
                        indexText = "".join(list(filter(str.isdigit, str(dataItem['_id']).strip())))
                        if indexText == str(dataItem['_id']).strip():
                            valueText = str(dataItem['_id']).strip()
                        else:
                            valueText = str(dataItem['_id']).strip().replace(indexText, '')
                        if str(dataItem['_id']).strip().find(indexText) < 0:
                            charEnd = -1
                            if str(dataItem['_id']).strip().find(indexText[0:1]) == 0:
                                for charIndex in range(len(indexText)):
                                    if str(dataItem['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                        charEnd = charIndex
                                        break
                            if charEnd != -1:
                                indexText = indexText[0:charEnd]
                                valueText = str(dataItem['_id']).strip()[charEnd:len(str(dataItem['_id']).strip())]
                            else:
                                indexText = ''
                                valueText = str(dataItem['_id']).strip()
                        if indexText == "" : indexText = str(index)
                        rowIDValuesJson1.append({indexText: valueText})
                    else:
                        rowIDValuesJson1.append({str(index + 1): str(dataItem['_id']).strip()})
                    rowIDValuesJson2.append({str(dataItem['_id']).strip(): str(dataItem['_id']).strip()})
                    if not is_number(str(dataItem['_id']).strip()):
                        numFlag = False
                    if idText == "": sortText.append(str(dataItem['_id']).strip())
                    index += 1
            if numFlag:
                rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson2)))
            else:
                if len(sortText) > 0:
                    sortTextArray = sorted(sortText, key=lazy_pinyin)
                    rowIDValues = json.dumps({str(index + 1): value for index, value in enumerate(sortTextArray)})
                else:
                    rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson1)))
        if not columnIDValues or str(columnIDValues).strip() == "" or not check_json_format(str(columnIDValues).strip()):
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            RowData = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, ColumnID, "", CasesCondition)
            #retFrequency.close()
            index, columnIDValuesJson1, columnIDValuesJson2, numFlag, indexText, valueText = 0, [], [], True, "", ""
            sortText = []
            if idText != "": index = 999999000
            for dataItem in RowData:
                if len(variableMore) > 0:
                    validFlag,text, numFlag = False, [], False
                    if dataItem['_id'] is not None:
                        validFlag = True
                        for variableItem in variableMore:
                            if dataItem['_id'].get(variableItem) is None or \
                                    str(dataItem['_id'].get(variableItem)).strip() == '' or \
                                    str(dataItem['_id'].get(variableItem)).strip() == 'None' or \
                                    str(dataItem['_id'].get(variableItem)).strip() == 'null':
                                validFlag = False
                    if validFlag:
                        for variableItem in variableMore:
                            text.append(str(dataItem['_id'].get(variableItem)).strip())
                        columnIDValuesJson1.append({str(index + 1): text})
                        index += 1
                else:
                    if dataItem['_id'] is not None and str(dataItem['_id']).strip() != '' and \
                            str(dataItem['_id']).strip() != 'None' and str(dataItem['_id']).strip() != 'null':
                        if str(dataItem['_id']).strip().find('年') > 0 and str(dataItem['_id']).strip().find('月') > 0:
                            idText = ""
                        if idText != "" and "".join(list(filter(str.isdigit, str(dataItem['_id']).strip()))) != "":
                            indexText = "".join(list(filter(str.isdigit, str(dataItem['_id']).strip())))
                            if indexText == str(dataItem['_id']).strip():
                                valueText = str(dataItem['_id']).strip()
                            else:
                                valueText = str(dataItem['_id']).strip().replace(indexText, '')
                            if str(dataItem['_id']).strip().find(indexText) < 0:
                                charEnd = -1
                                if str(dataItem['_id']).strip().find(indexText[0:1]) == 0:
                                    for charIndex in range(len(indexText)):
                                        if str(dataItem['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                            charEnd = charIndex
                                            break
                                if charEnd != -1:
                                    indexText = indexText[0:charEnd]
                                    valueText = str(dataItem['_id']).strip()[charEnd:len(str(dataItem['_id']).strip())]
                                else:
                                    indexText = ''
                                    valueText = str(dataItem['_id']).strip()
                            if indexText == "": indexText = str(index + 1)
                            columnIDValuesJson1.append({indexText: valueText})
                        else:
                            columnIDValuesJson1.append({str(index + 1): str(dataItem['_id']).strip()})
                        columnIDValuesJson2.append({str(dataItem['_id']).strip(): str(dataItem['_id']).strip()})
                        if not is_number(str(dataItem['_id']).strip()):
                            numFlag = False
                        if idText == "": sortText.append(str(dataItem['_id']).strip())
                        index += 1
            if numFlag:
                columnIDValues = json.dumps(dict(ChainMap(*columnIDValuesJson2)))
            else:
                if len(sortText) > 0:
                    sortTextArray = sorted(sortText, key=lazy_pinyin)
                    columnIDValues = json.dumps({str(index + 1): value for index, value in enumerate(sortTextArray)})
                else:
                    columnIDValues = json.dumps(dict(ChainMap(*columnIDValuesJson1)))
        jsonValue = dict(head=[], detailValid=[], detailValidgroup=[], chiSquare=[])
        allCount,validCount,validP,missingP = 0.0,0.0,0.0,0.0
        DataSource = pd.DataFrame(list(DataCount)).values.tolist()
        if isinstance(DataSource, list) and len(DataSource) > 0:
            for dataItem in DataSource:
                allCount += float(dataItem[1])
            for key,value in json.loads(rowIDValues).items():
                validN = 0.0
                for k,v in json.loads(columnIDValues).items():
                    for dataItem in DataSource:
                        validFlag = False
                        if isinstance(v, list):
                            if RowID in dataItem[0].keys() and \
                                    (str(key).strip() == str(dataItem[0].get(RowID)).strip() or
                                     str(value).strip() == str(dataItem[0].get(RowID)).strip() or
                                     (str(key).strip() + str(value).strip() == str(dataItem[0].get(RowID)).strip() and
                                      str(key).strip() != str(value).strip())):
                                validFlag = True
                                for variableIndex, variableItem in enumerate(variableMore):
                                    if variableItem in dataItem[0].keys():
                                        if str(v[variableIndex]).strip() != str(dataItem[0].get(variableItem)).strip():
                                            validFlag = False
                        else:
                            if RowID in dataItem[0].keys() and \
                                    (str(key).strip() == str(dataItem[0].get(RowID)).strip() or
                                     str(value).strip() == str(dataItem[0].get(RowID)).strip() or
                                     (str(key).strip() + str(value).strip() == str(dataItem[0].get(RowID)).strip() and
                                      str(key).strip() != str(value).strip())) and \
                                ColumnID in dataItem[0].keys() and \
                                    (str(k).strip() == str(dataItem[0].get(ColumnID)).strip() or
                                     str(v).strip() == str(dataItem[0].get(ColumnID)).strip() or
                                     (str(k).strip() + str(v).strip() == str(dataItem[0].get(ColumnID)).strip() and
                                      str(k).strip() != str(v).strip())):
                                validFlag = True
                        if validFlag:
                            validN += float(dataItem[1])
                            validCount += float(dataItem[1])
                            break
                for k,v in json.loads(columnIDValues).items():
                    for dataItem in DataSource:
                        num, per, validFlag = float(dataItem[1]), 0, False
                        if isinstance(v, list):
                            if RowID in dataItem[0].keys() and \
                                    (str(key).strip() == str(dataItem[0].get(RowID)).strip() or
                                     str(value).strip() == str(dataItem[0].get(RowID)).strip() or
                                     (str(key).strip() + str(value).strip() == str(dataItem[0].get(RowID)).strip() and
                                      str(key).strip() != str(value).strip())):
                                validFlag = True
                                for variableIndex, variableItem in enumerate(variableMore):
                                    if variableItem in dataItem[0].keys():
                                        if str(v[variableIndex]).strip() != str(dataItem[0].get(variableItem)).strip():
                                            validFlag = False
                        else:
                            if RowID in dataItem[0].keys() and \
                                    (str(key).strip() == str(dataItem[0].get(RowID)).strip() or
                                     str(value).strip() == str(dataItem[0].get(RowID)).strip() or
                                     (str(key).strip() + str(value).strip() == str(dataItem[0].get(RowID)).strip() and
                                      str(key).strip() != str(value).strip())) and \
                                ColumnID in dataItem[0].keys() and \
                                    (str(k).strip() == str(dataItem[0].get(ColumnID)).strip() or
                                     str(v).strip() == str(dataItem[0].get(ColumnID)).strip() or
                                     (str(k).strip() + str(v).strip() == str(dataItem[0].get(ColumnID)).strip() and
                                      str(k).strip() != str(v).strip())):
                                validFlag = True
                        if validFlag:
                            if validN != 0:
                                per = float(num) / float(validN)
                            jsonValue["detailValid"].append(dict(valueRow=int(float(key)), textRow=value,
                                                                 valueColumn=int(float(k)),
                                                                 textColumn=v,
                                                                 total=validN, count=num, validpercent=per))
                            break
                option,groupNum = "",0.0
                if Group and Group != "":
                    groupsplit=Group.split(',')
                    for k,v in json.loads(columnIDValues).items():
                        if k in groupsplit:
                            if option!="":
                                option+=","
                            option+=v
                    insertFlag=False
                    for dataItem in DataSource:
                        if RowID in dataItem[0].keys() and key==dataItem[0].get(RowID) and \
                                ColumnID in dataItem[0].keys() and dataItem[0].get(ColumnID) in groupsplit:
                            groupNum+=float(dataItem[1])
                            insertFlag=True
                    per=0.0
                    if validN!=0:
                        per=float(groupNum)/float(validN)
                    if insertFlag:
                        jsonValue["detailValidgroup"].append(dict(valueRow=int(float(key)), textRow=value,
                                                                  valueColumn="", textColumn=option,
                                                                  total=validN, count=int(groupNum), validpercent=per))
        jsonValue["detailValid"].sort(key=lambda x:(x["valueRow"], x["valueColumn"]), reverse=False)
        if str(deleteRow) != "" and isinstance(deleteRow, dict):
            detailRemove = []
            for index, detailItem in enumerate(jsonValue["detailValid"]):
                removeFlag = False
                for key, value in deleteRow.items():
                    if detailItem[key] < value:
                        removeFlag = True
                if removeFlag:
                    detailRemove.append(detailItem)
            for index, detailItem in enumerate(detailRemove):
                jsonValue["detailValid"].remove(detailItem)
        if float(allCount) != 0:
            validP = float(validCount) / float(allCount)
            missingP = float(allCount - validCount) / float(allCount)
        jsonValue["head"].append(
            dict(rowVarLabel=rowVarLabel, columnVarLabel=columnVarLabel, RowID=RowID, ColumnID=ColumnID,
                 count=validCount, totalN=allCount, validN=validCount, validP=validP,
                 invalidN=allCount - validCount, invalidP=missingP, Chi_Square_text='', Chi_Square_q=0, Chi_Square_P=0,
                 pearson_text='', pearson_r=0, pearson_p_value=0, spearman_text='', spearman_r=0, spearman_p_value=0,
                 analysis="crosstabs"))
        chiSquare_Tests = chiSquareway.run(DataFrame(jsonValue["detailValid"]), "valueRow", "valueColumn")
        jsonValue["chiSquare"].append(chiSquare_Tests)
        if str(chiSquare_Tests).find("NAN") < 0:
            Chi_Square_P = '0.000'
            try:
                Chi_Square_P = formatdata(chiSquare_Tests['chisq'], 3)
            except Exception as e:
                pass
            if 0.01 <= chiSquare_Tests['P'] < 0.05: Chi_Square_P += '*'
            if 0.001 <= chiSquare_Tests['P'] < 0.01: Chi_Square_P += '**'
            if chiSquare_Tests['P'] < 0.001: Chi_Square_P += '***'
            jsonValue["head"][0]["Chi_Square_text"] = Chi_Square_P
            if math.isnan(chiSquare_Tests['chisq']):
                jsonValue["head"][0]["Chi_Square_q"] = None
            else:
                jsonValue["head"][0]["Chi_Square_q"] = chiSquare_Tests['chisq']
            if math.isnan(chiSquare_Tests['P']):
                jsonValue["head"][0]["Chi_Square_P"] = None
            else:
                jsonValue["head"][0]["Chi_Square_P"] = chiSquare_Tests['P']
        if (rowVarType == 'FLOAT' or rowVarType == 'BIGINT') and (columnVarType == 'FLOAT' or columnVarType == 'BIGINT'):
            correlation = mongoCorrelationAnalysisDetail(PlatID, mongodb_cfg, ProjID, dataSetId, [RowID, ColumnID],
                                                         None, dataSetIdFlag, CasesCondition)
            if isinstance(correlation, dict) and correlation and RowID in correlation.keys() and \
                    ColumnID in correlation[RowID].keys():
                if 'pearson_r' in correlation[RowID][ColumnID].keys() and \
                    'pearson_p_value' in correlation[RowID][ColumnID].keys():
                    pearson_r_p = '0.000'
                    try:
                        pearson_r_p  = formatdata(correlation[RowID][ColumnID]['pearson_r'], 3)
                    except Exception as e:
                        pass
                    if 0.01 <= correlation[RowID][ColumnID]['pearson_p_value'] < 0.05: pearson_r_p += '*'
                    if 0.001 <= correlation[RowID][ColumnID]['pearson_p_value'] < 0.01: pearson_r_p += '**'
                    if correlation[RowID][ColumnID]['pearson_p_value'] < 0.001: pearson_r_p += '***'
                    jsonValue["head"][0]["pearson_text"] = pearson_r_p
                    if math.isnan(correlation[RowID][ColumnID]['pearson_r']):
                        jsonValue["head"][0]["pearson_r"] = None
                    else:
                        jsonValue["head"][0]["pearson_r"] = correlation[RowID][ColumnID]['pearson_r']
                    if math.isnan(correlation[RowID][ColumnID]['pearson_p_value']):
                        jsonValue["head"][0]["pearson_p_value"] = None
                    else:
                        jsonValue["head"][0]["pearson_p_value"] = correlation[RowID][ColumnID]['pearson_p_value']
                if 'spearman_r' in correlation[RowID][ColumnID].keys() and \
                    'spearman_p_value' in correlation[RowID][ColumnID].keys():
                    spearman_r_p = '0.000'
                    try:
                        spearman_r_p  = formatdata(correlation[RowID][ColumnID]['spearman_r'], 3)
                    except Exception as e:
                        pass
                    if 0.01 <= correlation[RowID][ColumnID]['spearman_p_value'] < 0.05: spearman_r_p += '*'
                    if 0.001 <= correlation[RowID][ColumnID]['spearman_p_value'] < 0.01: spearman_r_p += '**'
                    if correlation[RowID][ColumnID]['spearman_p_value'] < 0.001: spearman_r_p += '***'
                    jsonValue["head"][0]["spearman_text"] = spearman_r_p
                    if math.isnan(correlation[RowID][ColumnID]['spearman_r']):
                        jsonValue["head"][0]["spearman_r"] = None
                    else:
                        jsonValue["head"][0]["spearman_r"] = correlation[RowID][ColumnID]['spearman_r']
                    if math.isnan(correlation[RowID][ColumnID]['spearman_p_value']):
                        jsonValue["head"][0]["spearman_p_value"] = None
                    else:
                        jsonValue["head"][0]["spearman_p_value"] = correlation[RowID][ColumnID]['spearman_p_value']
        return jsonValue
    except Exception as ex:
        return {}


# -----<交叉表：多重响应单一变量>----- #
def mongo_crosstabsMultiple_logic(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID, phrase, deleteRow, idText,
                                  weight, dataSetIdFlag, CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion=dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID,ProjID,dataSetId)
        #retVersion.close()
    versionNum = None

    retMeta = select_meta_variable(mongodb_cfg)
    DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum,RowID)
    DataColumnID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId,versionNum, ColumnID)
    #retMeta.close()
    retCrosstabs = mongoCrosstabsModel(mongodb_cfg)
    DataCount = retCrosstabs.CrosstabsCountMongo(dataSetFlag, ProjID, dataSetId, versionNum,RowID, ColumnID,
                                                 weight, CasesCondition)
    #retCrosstabs.close()
    rowVarLabel, columnVarLabel, rowIDValues, columnIDValues, numFlag  = "", "", None, None, True
    for rowItem in DataRowID :
        if 'varValues' in rowItem: rowIDValues = rowItem['varValues']
        if 'varLabel' in rowItem: rowVarLabel= rowItem['varLabel']
    for rowItem in DataColumnID :
        if 'varValues' in rowItem: columnIDValues = rowItem['varValues']
        if 'varLabel' in rowItem: columnVarLabel = rowItem['varLabel']
    try:
        if not rowIDValues or str(rowIDValues).strip() == "" or not check_json_format(str(rowIDValues).strip()):
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            RowData = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, "", CasesCondition)
            #retFrequency.close()
            index, rowIDValuesJson1, rowIDValuesJson2, numFlag, indexText, valueText, sortText = \
                0, [], [], True, "", "", []
            if idText != "":index = 999999000
            for dataItem in RowData:
                if dataItem['_id'] is not None and str(dataItem['_id']).strip() != '' and \
                        str(dataItem['_id']).strip() != 'None' and str(dataItem['_id']).strip() != 'null':
                    if idText != "" and "".join(list(filter(str.isdigit, str(dataItem['_id']).strip()))) != "":
                        indexText = "".join(list(filter(str.isdigit, str(dataItem['_id']).strip())))
                        if indexText == str(dataItem['_id']).strip():
                            valueText = str(dataItem['_id']).strip()
                        else:
                            valueText = str(dataItem['_id']).strip().replace(indexText, '')
                        if str(dataItem['_id']).strip().find(indexText) < 0:
                            charEnd = -1
                            if str(dataItem['_id']).strip().find(indexText[0:1]) == 0:
                                for charIndex in range(len(indexText)):
                                    if str(dataItem['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                        charEnd = charIndex
                                        break
                            if charEnd != -1:
                                indexText = indexText[0:charEnd]
                                valueText = str(dataItem['_id']).strip()[charEnd:len(str(dataItem['_id']).strip())]
                            else:
                                indexText = ''
                                valueText = str(dataItem['_id']).strip()
                        if indexText == "" : indexText = str(index)
                        rowIDValuesJson1.append({indexText: valueText})
                    else:
                        rowIDValuesJson1.append({str(index + 1): str(dataItem['_id']).strip()})
                    rowIDValuesJson2.append({str(dataItem['_id']).strip(): str(dataItem['_id']).strip()})
                    if not is_number(str(dataItem['_id']).strip()):
                        numFlag = False
                    if idText == "": sortText.append(str(dataItem['_id']).strip())
                    index += 1
            if numFlag:
                rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson2)))
            else:
                if len(sortText) > 0:
                    sortTextArray = sorted(sortText, key=lazy_pinyin)
                    rowIDValues = json.dumps({str(index + 1): value for index, value in enumerate(sortTextArray)})
                else:
                    rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson1)))
        if not columnIDValues or str(columnIDValues).strip() == "" or not check_json_format(str(columnIDValues).strip()):
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            DataCountVar = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, ColumnID, weight,
                                                   CasesCondition)
            # retFrequency.close()
            keys, keys2, values, index, numFlag = [], [], [], 0, True
            for j in DataCountVar:
                if j['_id'] is not None and str(j['_id']).strip() != "" and str(j['_id']).strip() != "None":
                    if "," in j['_id']:
                        varstr = str(j['_id']).strip().split(",")
                    elif "." in j['_id']:
                        varstr = str(j['_id']).strip().split(".")
                    else:
                        varstr = [str(j['_id']).strip()]
                    for itemVar in varstr:
                        if itemVar not in values:
                            if not is_number(itemVar): numFlag = False
                            index += 1
                            keys.append(str(index))
                            keys2.append(itemVar)
                            values.append(itemVar)
            if not numFlag:
                columnIDValues = json.dumps(dict(zip(keys, values)))
            else:
                columnIDValues = json.dumps(dict(zip(keys2, values)))
        columnIDValues = json.loads(columnIDValues)
        if isinstance(phrase, dict) and numFlag:
            for key, value in columnIDValues.items():
                for k, v in phrase.items():
                    if key == k:
                        columnIDValues[key] = v
                        break
        jsonValue = dict(head=[], detailValid=[])
        allCount,validCount,validP,missingP,valueCount,delimiter = 0.0,0.0,0.0,0.0,{},"."
        DataSource = pd.DataFrame(list(DataCount)).values.tolist()
        if isinstance(DataSource, list) and len(DataSource) > 0:
            for dataItem in DataSource:
                allCount += float(dataItem[1])
            for key,value in json.loads(rowIDValues).items():
                validN, caseN = 0.0, 0.0
                for k, v in columnIDValues.items():
                    valueCount[k] = 0.0
                for dataItem in DataSource:
                    if "," in str(dataItem[0].get(ColumnID)).strip():
                        delimiter = ","
                    if RowID in dataItem[0].keys() and \
                            (str(key).strip() == str(dataItem[0].get(RowID)).strip() or
                             str(value).strip() == str(dataItem[0].get(RowID)).strip() or
                             (str(key).strip() + str(value).strip() == str(dataItem[0].get(RowID)).strip() and
                              str(key).strip() != str(value).strip())):
                        if not (not dataItem[0].get(ColumnID) or str(dataItem[0].get(ColumnID)).strip() == "" or
                            str(dataItem[0].get(ColumnID)).strip() == "None"):
                            caseN += float(dataItem[1])
                            for k,v in columnIDValues.items():
                                if (numFlag and (delimiter + str(dataItem[0].get(ColumnID)) + delimiter).
                                        find(delimiter + str(k) + delimiter) >= 0) or \
                                        (not numFlag and (delimiter + str(dataItem[0].get(ColumnID)) + delimiter).
                                        find(delimiter + str(v) + delimiter) >= 0):
                                    valueCount[k] += float(dataItem[1])
                                    validN += float(dataItem[1])
                            validCount += float(dataItem[1])
                for k,v in columnIDValues.items():
                    num, caseP, validP = float(valueCount[k]), 0.0, 0.0
                    if num > 0:
                        if caseN != 0:
                            caseP = num / caseN
                        if validN != 0:
                            validP = num / validN
                        jsonValue["detailValid"].append(dict(valueRow=int(float(key)), textRow=value,
                                                             valueColumn=int(float(k)), textColumn=v, total=validN,
                                                             count=int(num), casepercent=caseP, validpercent=validP))
        jsonValue["detailValid"].sort(key=lambda x:(x["valueRow"], x["valueColumn"]), reverse=False)
        if str(deleteRow) != "" and isinstance(deleteRow, dict):
            detailRemove = []
            for index, detailItem in enumerate(jsonValue["detailValid"]):
                removeFlag = False
                for key, value in deleteRow.items():
                    if detailItem[key] < value:
                        removeFlag = True
                if removeFlag:
                    detailRemove.append(detailItem)
            for index, detailItem in enumerate(detailRemove):
                jsonValue["detailValid"].remove(detailItem)
        if float(allCount) != 0:
            validP = float(validCount) / float(allCount)
            missingP = float(allCount - validCount) / float(allCount)
        jsonValue["head"].append(
            dict(rowVarLabel=rowVarLabel, columnVarLabel=columnVarLabel, RowID=RowID, ColumnID=ColumnID,
                 count=validCount, totalN=allCount, validN=validCount, validP=validP,
                 invalidN=allCount - validCount, invalidP=missingP, analysis="crosstabsMultiple"))
        return jsonValue
    except Exception as ex:
        return {}


# -----<交叉表：多重响应二分法>----- #
def mongo_crosstabsMultiple2_logic(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID, phrase, OptionID, deleteRow,
                                   idText, weight, dataSetIdFlag, CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion=dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID,ProjID,dataSetId)
        #retVersion.close()
    versionNum = None

    retMeta = select_meta_variable(mongodb_cfg)
    DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, RowID)
    #retMeta.close()
    retCrosstabs = mongoCrosstabsModel(mongodb_cfg)
    #retCrosstabs.close()
    rowVarLabel,rowIDValues = "",  None
    for rowItem in DataRowID :
        if 'varValues' in rowItem: rowIDValues = rowItem['varValues']
        if 'varLabel' in rowItem: rowVarLabel= rowItem['varLabel']
    try:
        if not rowIDValues or str(rowIDValues).strip() == "" or not check_json_format(str(rowIDValues).strip()):
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            RowData = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, "", CasesCondition)
            #retFrequency.close()
            index, rowIDValuesJson1, rowIDValuesJson2, numFlag, indexText, valueText, sortText = \
                0, [], [], True, "", "", []
            if idText != "":index = 999999000
            for dataItem in RowData:
                if dataItem['_id'] is not None and str(dataItem['_id']).strip() != '' and \
                        str(dataItem['_id']).strip() != 'None' and str(dataItem['_id']).strip() != 'null':
                    if idText != "" and "".join(list(filter(str.isdigit, str(dataItem['_id']).strip()))) != "":
                        indexText = "".join(list(filter(str.isdigit, str(dataItem['_id']).strip())))
                        if indexText == str(dataItem['_id']).strip():
                            valueText = str(dataItem['_id']).strip()
                        else:
                            valueText = str(dataItem['_id']).strip().replace(indexText, '')
                        if str(dataItem['_id']).strip().find(indexText) < 0:
                            charEnd = -1
                            if str(dataItem['_id']).strip().find(indexText[0:1]) == 0:
                                for charIndex in range(len(indexText)):
                                    if str(dataItem['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                        charEnd = charIndex
                                        break
                            if charEnd != -1:
                                indexText = indexText[0:charEnd]
                                valueText = str(dataItem['_id']).strip()[charEnd:len(str(dataItem['_id']).strip())]
                            else:
                                indexText = ''
                                valueText = str(dataItem['_id']).strip()
                        if indexText == "" : indexText = str(index)
                        rowIDValuesJson1.append({indexText: valueText})
                    else:
                        rowIDValuesJson1.append({str(index + 1): str(dataItem['_id']).strip()})
                    rowIDValuesJson2.append({str(dataItem['_id']).strip(): str(dataItem['_id']).strip()})
                    if not is_number(str(dataItem['_id']).strip()):
                        numFlag = False
                    if idText == "": sortText.append(str(dataItem['_id']).strip())
                    index += 1
            if numFlag:
                rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson2)))
            else:
                if len(sortText) > 0:
                    sortTextArray = sorted(sortText, key=lazy_pinyin)
                    rowIDValues = json.dumps({str(index + 1): value for index, value in enumerate(sortTextArray)})
                else:
                    rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson1)))
        if str(ColumnID).find(",") > -1:
            VariableOptionID = str(ColumnID).strip().split(',')
        else:
            if not isinstance(OptionID, list):
                fromTO = str(OptionID).strip().split('-')
                OptionID = []
                if float(fromTO[0]) == 0:
                    for n in range(int(fromTO[1])):
                        OptionID.append(str(n))
                else:
                    for n in range(int(fromTO[1])):
                        OptionID.append(str(n + 1))
            VariableOptionID = []
            for itemOpt in OptionID:
                VariableOptionID.append(ColumnID + str(itemOpt))
        allCount, validN, invalidN, valueCount, varLabel, optionValue, caseListN, OptionIDKey, OptionIDValue = \
            0.0, {}, {}, {}, {}, {}, {}, [], []
        caseCount, validCount = 0.0, 0.0
        for itemInd, itemOpt in enumerate(VariableOptionID):
            OptionIDKey.append(itemInd)
            OptionIDValue.append(0.0)
        for key, value in json.loads(rowIDValues).items():
            validN[key] = 0
            invalidN[key] = 0
            valueCount[key] = dict(zip(OptionIDKey, OptionIDValue))
            caseListN[key] = dict(zip(OptionIDKey, OptionIDValue))
        for itemInd, itemOpt in enumerate(VariableOptionID):
            retMeta = select_meta_variable(mongodb_cfg)
            DataVariable = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, str(itemOpt))
            varLabel[itemInd] = ""
            for rowItem in DataVariable:
                if 'varLabel' in rowItem: varLabel[itemInd] = rowItem['varLabel']
            if isinstance(phrase, list) and len(phrase) > itemInd:
                varLabel[itemInd] = phrase[itemInd]
            if is_number_int(itemOpt):
                optionValue[itemInd] = itemOpt
            else:
                optionValue[itemInd] = str(itemInd + 1)
            DataCount = retCrosstabs.CrosstabsCountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, str(itemOpt),
                                                         weight, CasesCondition)
            DataSource = pd.DataFrame(list(DataCount)).values.tolist()
            for key, value in json.loads(rowIDValues).items():
                for dataItem in DataSource:
                    if itemInd == 0: allCount += float(dataItem[1])
                    if RowID in dataItem[0].keys() and \
                            (str(key).strip() == str(dataItem[0].get(RowID)).strip() or
                             str(value).strip() == str(dataItem[0].get(RowID)).strip() or
                             (str(key).strip() + str(value).strip() == str(dataItem[0].get(RowID)).strip() and
                              str(key).strip() != str(value).strip())):
                        if str(dataItem[0].get(itemOpt)).strip() != '' and \
                            ((is_number(str(dataItem[0].get(itemOpt)).strip()) and
                              float(str(dataItem[0].get(itemOpt)).strip()) >= 0)
                                or str(dataItem[0].get(itemOpt)).strip() == '勾选' or
                             str(dataItem[0].get(itemOpt)).strip() == '未勾选'):
                            caseListN[key][itemInd] += float(dataItem[1])
                            if (is_number(str(dataItem[0].get(itemOpt)).strip()) and
                                float(str(dataItem[0].get(itemOpt)).strip()) > 0) or \
                                    str(dataItem[0].get(itemOpt)).strip() == '勾选':
                                valueCount[key][itemInd] += float(dataItem[1])
                                validN[key] += float(dataItem[1])

        jsonValue = dict(head=[], detailValid=[])
        CasesCondition0 = copy.deepcopy(CasesCondition)
        for itemOpt in VariableOptionID:
            if isinstance(CasesCondition0, list):
                CasesCondition0[0].append(str(itemOpt))
                CasesCondition0[1].append("0")
            else:
                CasesCondition0 = [[str(itemOpt)], ["0"]]
        DataCount = retCrosstabs.CrosstabsCountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID,
                                                     str(VariableOptionID[0]), weight, CasesCondition0)
        DataSource = pd.DataFrame(list(DataCount)).values.tolist()
        for key, value in json.loads(rowIDValues).items():
            caseN = 0
            for caseKey, caseValue in caseListN[key].items():
                if caseN < caseValue: caseN = caseValue
            for dataItem in DataSource:
                if RowID in dataItem[0].keys() and \
                        (str(key).strip() == str(dataItem[0].get(RowID)).strip() or
                         str(value).strip() == str(dataItem[0].get(RowID)).strip() or
                         (str(key).strip() + str(value).strip() == str(dataItem[0].get(RowID)).strip() and
                          str(key).strip() != str(value).strip())):
                    caseN = caseN - float(dataItem[1])
            caseCount += caseN
            validCount += validN[key]
            for k, v in valueCount[key].items():
                num, caseP, validP = float(v), 0.0, 0.0  # num=某个多选变量对应的响应数量
                if num > 0:
                    if caseN != 0:
                        caseP = float(v) / caseN  # caseP=个案百分比
                    if validN[key] != 0:
                        validP = float(v) / validN[key]  # validP=响应百分比
                    jsonValue["detailValid"].append(dict(valueRow=float(key), textRow=value,
                                                         valueColumn=int("".join(list(filter(str.isdigit, optionValue[k])))),
                                                         textColumn=varLabel[k], total=validN[key], count=int(num),
                                                         casepercent=caseP, validpercent=validP))

        jsonValue["detailValid"].sort(key=lambda x:(x["valueRow"], x["valueColumn"]), reverse=False)
        if str(deleteRow) != "" and isinstance(deleteRow, dict):
            detailRemove = []
            for index, detailItem in enumerate(jsonValue["detailValid"]):
                removeFlag = False
                for key, value in deleteRow.items():
                    if detailItem[key] < value:
                        removeFlag = True
                if removeFlag:
                    detailRemove.append(detailItem)
            for index, detailItem in enumerate(detailRemove):
                jsonValue["detailValid"].remove(detailItem)
        validP, missingP = 0.0, 0.0
        if float(allCount) != 0:
            validP = float(validCount) / float(allCount)
            missingP = float(allCount - validCount) / float(allCount)
        jsonValue["head"].append(
            dict(rowVarLabel=rowVarLabel, columnVarLabel='', RowID=RowID, ColumnID=str(VariableOptionID),
                 count=validCount, totalN=allCount, validN=validCount, validP=validP,
                 invalidN=allCount - validCount, invalidP=missingP, analysis="crosstabsMultiple"))
        return jsonValue
    except Exception as ex:
        return {}


# -----<比较均值>----- #
def mongo_comparemeans_logic(PlatID, mongodb_cfg, ProjID,dataSetId,RowID,ColumnID,VariableIDgroup, optionIDgroup,
                             IntervalRange,denominatorData,deleteRow,idText, weight, formula, dataSetIdFlag,
                             percentileRank, replaceText, CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
        #retVersion.close()
    versionNum = None

    rowVarLabel, columnVarLabel, rowIDValues, anovaArgs, rowIDValuesFlag, replaceFrom, replaceTo = \
        "", "", None, [], True, [], []
    if replaceText is None or not isinstance(replaceText, list):
        replaceText = []
    for i in range(len(replaceText)):
        if isinstance(replaceText[i][0], float):
            replaceFrom.append(replaceText[i][0])
        else:
            replaceFrom.append(float(str(replaceText[i][0])))
        if isinstance(replaceText[i][1], float):
            replaceTo.append(replaceText[i][1])
        else:
            if is_number(str(replaceText[i][1])):
                replaceTo.append(float(str(replaceText[i][1])))
            else:
                replaceTo.append('')
    retMeta = select_meta_variable(mongodb_cfg)
    DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum,RowID)
    if ColumnID != "":
        DataColumnID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId,versionNum, ColumnID)
        for rowItem in DataColumnID:
            if 'varLabel' in rowItem: columnVarLabel = rowItem['varLabel']
    #retMeta.close()
    for rowItem in DataRowID :
        if 'varValues' in rowItem:rowIDValues = rowItem['varValues']
        if 'varLabel' in rowItem: rowVarLabel = rowItem['varLabel']
    try:
        if not rowIDValues or str(rowIDValues).strip() == "" or not check_json_format(str(rowIDValues).strip()):
            rowIDValuesFlag = False
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            RowData = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, "", CasesCondition)
            #retFrequency.close()
            index, rowIDValuesJson1, rowIDValuesJson2, numFlag, indexText, valueText, sortText = \
                0, [], [], True, "", "", []
            if idText != "": index = 999999000
            for dataItem in RowData:
                if dataItem['_id'] is not None and str(dataItem['_id']).strip() != '' and \
                        str(dataItem['_id']).strip() != 'None' and str(dataItem['_id']).strip() != 'null':
                    if idText != "" and "".join(list(filter(str.isdigit, str(dataItem['_id']).strip()))) != "":
                        indexText = "".join(list(filter(str.isdigit, str(dataItem['_id']).strip())))
                        if indexText == str(dataItem['_id']).strip():
                            valueText = str(dataItem['_id']).strip()
                        else:
                            valueText = str(dataItem['_id']).strip().replace(indexText, '')
                        if str(dataItem['_id']).strip().find(indexText) < 0:
                            charEnd = -1
                            for charIndex in range(len(indexText)):
                                if str(dataItem['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                    charEnd = charIndex
                                    break
                            indexText = indexText[0:charEnd]
                            valueText = str(dataItem['_id']).strip()[charEnd:len(str(dataItem['_id']).strip())]
                        if indexText == "": indexText = str(index)
                        rowIDValuesJson1.append({indexText: valueText})
                    else:
                        rowIDValuesJson1.append({str(index + 1): str(dataItem['_id']).strip()})
                    rowIDValuesJson2.append({str(dataItem['_id']).strip(): str(dataItem['_id']).strip()})
                    if not is_number(str(dataItem['_id']).strip()):
                        numFlag = False
                    if idText == "": sortText.append(str(dataItem['_id']).strip())
                    index += 1
            if numFlag:
                rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson2)))
            else:
                if len(sortText) > 0:
                    sortTextArray = sorted(sortText, key=lazy_pinyin)
                    rowIDValues = json.dumps({str(index + 1): value for index, value in enumerate(sortTextArray)})
                else:
                    rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson1)))
        jsonValue = dict(head=[], detailValid=[], variance=[])
        totalN,validN,validP,invalidP,sumTotal = 0.0,0.0,0.0,0.0,0.0
        if str(ColumnID).strip() != "":
            retCompareMeans = mongoCompareMeansModel(mongodb_cfg)
            DataCount = retCompareMeans.CompareSumMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, ColumnID,
                                                        CasesCondition)
            #retCompareMeans.close()
            for dataItem in DataCount:
                totalN += float(dataItem['count'])
                if isinstance(IntervalRange, list):
                    if RowID in dataItem['_id'].keys() and ColumnID in dataItem['_id'].keys() and \
                            str(dataItem['_id'].get(RowID)).strip()!='' and str(dataItem['_id'].get(ColumnID)).strip()!='' \
                            and float(IntervalRange[0]) <= float(dataItem['_id'].get(ColumnID)) <= float(IntervalRange[1]):
                        validN += float(dataItem["count"])
                else:
                    if RowID in dataItem['_id'].keys() and ColumnID in dataItem['_id'].keys() and \
                            str(dataItem['_id'].get(RowID)).strip()!='' and str(dataItem['_id'].get(ColumnID)).strip()!='':
                        validN += float(dataItem["count"])
        for key, value in json.loads(rowIDValues).items():
            if rowIDValuesFlag:
                retCompareMeans = mongoCompareMeansModel(mongodb_cfg)
                sortData = retCompareMeans.sortDataMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, ColumnID,
                                                         key, VariableIDgroup, weight,CasesCondition)
                #retCompareMeans.close()
            else:
                if idText != "":
                    retCompareMeans = mongoCompareMeansModel(mongodb_cfg)
                    sortData = retCompareMeans.sortDataMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, ColumnID,
                                                             key + value, VariableIDgroup, weight, CasesCondition)
                    #retCompareMeans.close()
                else:
                    retCompareMeans = mongoCompareMeansModel(mongodb_cfg)
                    sortData = retCompareMeans.sortDataMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, ColumnID,
                                                             value, VariableIDgroup, weight,CasesCondition)
                    #retCompareMeans.close()
            DataSort,passRate,DataWeight = [],0.0,[]
            if str(ColumnID).strip() != "":
                for j in sortData:
                    if str(j[ColumnID]).strip() != '' and is_number(str(j[ColumnID]).strip()):
                        if weight != "" and str(j[weight]).strip() != '' and is_number(str(j[weight]).strip()):
                            DataWeight.append(float(str(j[weight]).strip()))
                        if float(str(j[ColumnID]).strip()) in replaceFrom:
                            if isinstance(replaceTo[replaceFrom.index(float(str(j[ColumnID]).strip()))], float):
                                DataSort.append(replaceTo[replaceFrom.index(float(str(j[ColumnID]).strip()))])
                        else:
                            DataSort.append(float(str(j[ColumnID]).strip()))
                if isinstance(IntervalRange, list):
                    DataSortCopy = DataSort[:]
                    for num in DataSort:
                        if num < float(IntervalRange[0]) or float(IntervalRange[0]) > float(IntervalRange[1]):
                            DataSortCopy.remove(num)
                    DataSort = DataSortCopy
                if len(DataSort) > 0:
                    for num in DataSort:
                        anovaArgs.append([num, float(key)])
            else:
                for dataItem in sortData:
                    num = 0.0
                    if isinstance(optionIDgroup, str) and optionIDgroup == "":
                        for variableItem in VariableIDgroup:
                            if dataItem[variableItem] is not None and str(dataItem[variableItem]).strip() != "":
                                num += float(str(dataItem[variableItem]))
                    else:
                        for optionItem in optionIDgroup:
                            if 0 < int(optionItem) <= 150:
                                if dataItem[VariableIDgroup[0]] is not None and \
                                        str(dataItem[VariableIDgroup[0]]).strip() != "":
                                    num += float(str(dataItem[VariableIDgroup[0]])[int(optionItem) - 1:int(optionItem)])
                            else:
                                if dataItem[VariableIDgroup[1]] is not None and \
                                        str(dataItem[VariableIDgroup[1]]).strip() != "":
                                    num += float(str(dataItem[VariableIDgroup[1]])[int(optionItem) - 151:
                                                                                   int(optionItem) -150])
                    if num in replaceFrom:
                        if isinstance(replaceTo[replaceFrom.index(num)], float):
                            num = replaceTo[replaceFrom.index(num)]
                    if isinstance(IntervalRange, list):
                        if len(IntervalRange) == 2:
                            if float(IntervalRange[0]) <= num <= float(IntervalRange[1]):
                                DataSort.append(num)
                        elif len(IntervalRange) == 3:
                            if float(IntervalRange[0]) < num <= float(IntervalRange[1]):
                                DataSort.append(num)
                        else:
                            if IntervalRange[3] == 1:
                                if float(IntervalRange[0]) < num < float(IntervalRange[1]):
                                    DataSort.append(num)
                            else:
                                if float(IntervalRange[0]) <= num < float(IntervalRange[1]):
                                    DataSort.append(num)
                    else:
                        DataSort.append(num)
                validN += len(DataSort)
            if len(DataSort)>0:
                maxData = np.max(DataSort)
                cnt = len(DataSort)
                sumData = np.sum(DataSort)
                minData = np.min(DataSort)
                series = pd.Series(data=DataSort)
                if len(DataWeight) > 0:
                    avg = np.average(DataSort, 0, DataWeight)
                    PercentArray = weighted_quantile(DataSort, [0.05, 0.1, 0.25, 0.75, 0.9, 0.95], DataWeight)
                    Percent5, Percent10, Percent25, Percent75, Percent90, Percent95 = \
                        PercentArray[0], PercentArray[1], PercentArray[2], PercentArray[3], PercentArray[4], PercentArray[5]
                    std = np.sqrt(np.cov(DataSort, aweights=DataWeight))
                else:
                    avg = np.average(DataSort)
                    std = np.std(DataSort, axis=0, ddof=1)                   # 数据只有1条会报错，除0错误，返回值nan
                    Percent5 = np.percentile(DataSort, 5, method="weibull")
                    Percent10 = np.percentile(DataSort, 10, method="weibull")
                    Percent25 = np.percentile(DataSort, 25, method="weibull")
                    Percent75 = np.percentile(DataSort, 75, method="weibull")
                    Percent90 = np.percentile(DataSort, 90, method="weibull")
                    Percent95 = np.percentile(DataSort, 95, method="weibull")
                if np.isnan(std): std = 0.0
                var = np.var(DataSort)
                mid = np.median(DataSort)
                mode = series.mode()[0]
                sumTotal += sumData
                if isinstance(denominatorData, list):
                    denominatorText = denominatorData[0]
                    denominatorValue = denominatorData[1]
                    denominator = 0
                    for denominatorIndex,denominatorItem in enumerate(denominatorText):
                        if denominatorItem == value:
                            denominator = denominatorValue[denominatorIndex]
                            break
                    if denominator > 0:
                        passRate = cnt / denominator
                jsonValue["detailValid"].append(
                    dict(valueRow=int(float(key)), textRow=value, sum=sumData, count=cnt, average=avg, max=maxData,
                         median=mid, min=minData, mode=mode, stdev=std, passRate=passRate, validpercent=0.0,
                         Percent5=Percent5,Percent10=Percent10, Percent25=Percent25,Percent75=Percent75,
                         Percent90=Percent90, Percent95=Percent95, variance=var))
        if sumTotal != 0:
            for detailItem in jsonValue["detailValid"]:
                detailItem["validpercent"] = detailItem["sum"] / sumTotal
        jsonValue["detailValid"].sort(key=lambda x:x["valueRow"], reverse=False)
        if str(deleteRow) != "" and isinstance(deleteRow, dict):
            detailRemove = []
            for index, detailItem in enumerate(jsonValue["detailValid"]):
                removeFlag = False
                for key, value in deleteRow.items():
                    if detailItem[key] < value:
                        removeFlag = True
                if removeFlag:
                    detailRemove.append(detailItem)
            for index, detailItem in enumerate(detailRemove):
                jsonValue["detailValid"].remove(detailItem)
        if totalN != 0:
            validP = validN / totalN
            invalidP = (totalN - validN) / totalN
        jsonValue["head"].append(dict(rowVarLabel=rowVarLabel, columnVarLabel=columnVarLabel,RowID=RowID,
                                      ColumnID=ColumnID, count=int(validN),totalN=int(totalN),validN=int(validN),
                                      validP=validP, length=len(jsonValue["detailValid"]),
                                      invalidN=int(totalN - validN),invalidP=invalidP, variance_text='', variance_F=0,
                                      variance_P=0, F_T_text='', F_T_value=0, F_T_P=0, analysis="comparemeans"))
        if percentileRank is not None:
            Condition2 = percentileRank['percentileRank']['condition']
            Condition2[0].append(RowID)
            Condition2[1].append('')
            for detailValidIndex, detailValidItem in enumerate(jsonValue["detailValid"]):
                if rowIDValuesFlag:
                    Condition2[1][len(Condition2[1]) - 1] = str(detailValidItem['valueRow'])
                else:
                    Condition2[1][len(Condition2[1]) - 1] = str(detailValidItem['textRow'])
                retCompareMeans = mongoMeansModel(mongodb_cfg)
                DataSum2 = retCompareMeans.SumMongoCompareAvg(dataSetFlag, ProjID, dataSetId, None,
                                                              percentileRank['percentileRank']['RowID'],
                                                              [ColumnID], optionIDgroup, "", Condition2)
                datasource2 = dataSourceSet(DataSum2, percentileRank['percentileRank']['RowID'], [ColumnID], '0', True)
                jsonValue["detailValid"][detailValidIndex]['percentileRank'] = \
                    getPercentileRank(detailValidItem, datasource2['detailValid'])
        if ColumnID != "":
            try:
                anovaArgsData = pd.DataFrame(anovaArgs, columns=[RowID, ColumnID])
                expr = '{}~C({})'.format(RowID, ColumnID)
                mod = ols(expr, data=anovaArgsData).fit()
                anova_table = sm.stats.anova_lm(mod)
                ret = {'df': list(anova_table.df),
                       'sum_sq': list(anova_table.sum_sq),
                       'mean_sq': list(anova_table.mean_sq),
                       'F': list(anova_table.F)[0],
                       'P': list(anova_table.values.T[-1])[0]}
            except Exception as e:
                ret = {"df": "NaN", "sum_sq": "NaN", "mean_sq": "Nan", "F": "NaN", "P": "NaN"}
            jsonValue["variance"].append(ret)
            if str(ret).find("NaN") < 0:
                try:
                    variance_P = formatdata(ret['F'], 2)
                except Exception as e:
                    variance_P = '0.00'
                if 0.01 <= ret['P'] < 0.05: variance_P += '*'
                if 0.001 <= ret['P'] < 0.01: variance_P += '**'
                if ret['P'] < 0.001: variance_P += '***'
                jsonValue["head"][0]["variance_text"] = variance_P
                jsonValue["head"][0]["F_T_text"] = variance_P
                if math.isnan(ret['F']):
                    jsonValue["head"][0]["variance_F"] = None
                    jsonValue["head"][0]["F_T_value"] = None
                else:
                    jsonValue["head"][0]["variance_F"] = ret['F']
                    jsonValue["head"][0]["F_T_value"] = ret['F']
                if math.isnan(ret['P']):
                    jsonValue["head"][0]["variance_P"] = None
                    jsonValue["head"][0]["F_T_P"] = None
                else:
                    jsonValue["head"][0]["variance_P"] = ret['P']
                    jsonValue["head"][0]["F_T_P"] = ret['P']
            if len(jsonValue["detailValid"]) == 2:
                if rowIDValuesFlag:
                    TtestData = mongoTtest_indResultDetail(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID,
                                                           jsonValue["detailValid"][0]["valueRow"],
                                                           jsonValue["detailValid"][1]["valueRow"],
                                                           dataSetIdFlag, CasesCondition)
                else:
                    TtestData = mongoTtest_indResultDetail(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID,
                                                           jsonValue["detailValid"][0]["textRow"],
                                                           jsonValue["detailValid"][1]["textRow"],
                                                           dataSetIdFlag, CasesCondition)
                if TtestData:
                    try:
                        F_T_P = formatdata(TtestData['t_sig'], 2)
                    except Exception as e:
                        F_T_P = '0.00'
                    if 0.01 <= TtestData['Sig'] < 0.05: F_T_P += '*'
                    if 0.001 <= TtestData['Sig'] < 0.01: F_T_P += '**'
                    if TtestData['Sig'] < 0.001: F_T_P += '***'
                    jsonValue["head"][0]["F_T_text"] = F_T_P
                    if math.isnan(TtestData['t_sig']):
                        jsonValue["head"][0]["F_T_value"] = None
                    else:
                        jsonValue["head"][0]["F_T_value"] = TtestData['t_sig']
                    if math.isnan(TtestData['Sig']):
                        jsonValue["head"][0]["F_T_P"] = None
                    else:
                        jsonValue["head"][0]["F_T_P"] = TtestData['Sig']
        if formula != '': calculation_formula(jsonValue["detailValid"], formula)
        return jsonValue
    except Exception as ex:
        return {}


def mongo_comparemeans2_logic(PlatID, mongodb_cfg, ProjID,dataSetId,RowID,VariableIDgroup,optionIDgroup,denominatorData,
                              deleteRow,idText,weight, formula, dataSetIdFlag, percentileRank, CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
        #retVersion.close()
    versionNum = None

    rowVarLabel, columnVarLabel, rowIDValues, anovaArgs, rowIDValuesFlag = "", "", {"0":""}, [], True
    #if RowID == 'category' and VariableIDgroup[0] == 'bykh':
        #testFlag = True
    if isinstance(RowID, str):
        retMeta = select_meta_variable(mongodb_cfg)
        DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum,RowID)
        #retMeta.close()
        for rowItem in DataRowID :
            if 'varValues' in rowItem: rowIDValues=rowItem['varValues']
            if 'varLabel' in rowItem: rowVarLabel = rowItem['varLabel']
        RowIDStr = RowID
    else:
        RowIDStr = RowID[0]
    if len(VariableIDgroup) == 1:
        retMeta = select_meta_variable(mongodb_cfg)
        DataColumnID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId,versionNum, VariableIDgroup[0])
        #retMeta.close()
        for rowItem in DataColumnID:
            if 'varLabel' in rowItem: columnVarLabel = rowItem['varLabel']
    try:
        if not rowIDValues or str(rowIDValues).strip() == "" or not check_json_format(str(rowIDValues).strip()):
            rowIDValuesFlag = False
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            RowData = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, "", CasesCondition)
            #retFrequency.close()
            index, rowIDValuesJson1, rowIDValuesJson2, numFlag, indexText, valueText, sortText = \
                0, [], [], True, "", "", []
            if idText != "": index = 999999000
            for dataItem in RowData:
                if dataItem['_id'] is not None and str(dataItem['_id']).strip() != '' and \
                        str(dataItem['_id']).strip() != 'None' and str(dataItem['_id']).strip() != 'null':
                    if idText != "" and "".join(list(filter(str.isdigit, str(dataItem['_id']).strip()))) != "":
                        indexText = "".join(list(filter(str.isdigit, str(dataItem['_id']).strip())))
                        if indexText == str(dataItem['_id']).strip():
                            valueText = str(dataItem['_id']).strip()
                        else:
                            valueText = str(dataItem['_id']).strip().replace(indexText, '')
                        if str(dataItem['_id']).strip().find(indexText) < 0:
                            charEnd = -1
                            for charIndex in range(len(indexText)):
                                if str(dataItem['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                    charEnd = charIndex
                                    break
                            indexText = indexText[0:charEnd]
                            valueText = str(dataItem['_id']).strip()[charEnd:len(str(dataItem['_id']).strip())]
                        if indexText == "": indexText = str(index)
                        rowIDValuesJson1.append({indexText: valueText})
                    else:
                        rowIDValuesJson1.append({str(index + 1): str(dataItem['_id']).strip()})
                    rowIDValuesJson2.append({str(dataItem['_id']).strip(): str(dataItem['_id']).strip()})
                    if not is_number(str(dataItem['_id']).strip()):
                        numFlag = False
                    if idText == "": sortText.append(str(dataItem['_id']).strip())
                    index += 1
            if numFlag:
                rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson2)))
            else:
                if len(sortText) > 0:
                    sortTextArray = sorted(sortText, key=lazy_pinyin)
                    rowIDValues = json.dumps({str(index + 1): value for index, value in enumerate(sortTextArray)})
                else:
                    rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson1)))
        jsonValue = dict(head=[], detailValid=[])
        totalN,validN,validP,invalidP,value, VariableItem, sumTotal = 0.0,0.0,0.0,0.0,0,"",0.0
        if weight != "": VariableIDgroup[0] = VariableIDgroup[0] + "*" + weight
        retCompareMeans = mongoMeansModel(mongodb_cfg)
        try:
            DataSum = retCompareMeans.SumMongoCompareAvg(dataSetFlag, ProjID, dataSetId, versionNum, RowID,
                                                         VariableIDgroup, optionIDgroup, weight, CasesCondition)
        except Exception as ex:
            DataSum = []
        #retCompareMeans.close()
        for dataItem in DataSum:
            text, avg, sumData, maxData, minData, cnt, passRate, VariableMore = None, 0, 0, 0, 0, 0, 0, []
            if dataItem['_id'][RowIDStr] is not None and \
                    str(dataItem['_id'][RowIDStr]).strip() != '' and \
                    str(dataItem['_id'][RowIDStr]).strip() != 'None' and str(dataItem['_id'][RowIDStr]).strip() != 'null':
                text = str(dataItem['_id'][RowIDStr]).strip()
            if VariableIDgroup[0].find("*") > -1:
                VariableMore = VariableIDgroup[0].split('*')
            elif VariableIDgroup[0].find("/") > -1:
                VariableMore = VariableIDgroup[0].split('/')
            elif VariableIDgroup[0].find("+") > -1:
                VariableMore = VariableIDgroup[0].split('+')
            elif VariableIDgroup[0].find("-") > -1:
                VariableMore = VariableIDgroup[0].split('-')
            if len(VariableMore) > 0:
                VariableItem = VariableMore[0]
            else:
                VariableItem = VariableIDgroup[0]
            if dataItem[VariableItem + 'avg'] is not None and \
                    str(dataItem[VariableItem + 'avg']).strip() != '':
                if weight != "":
                    avg = float(str(dataItem[VariableItem + 'sum']).strip()) / float(str(dataItem['count']).strip())
                else:
                    avg = float(str(dataItem[VariableItem + 'avg']).strip())
                sumData = float(str(dataItem[VariableItem + 'sum']).strip())
                maxData = float(str(dataItem[VariableItem + 'max']).strip())
                minData = float(str(dataItem[VariableItem + 'min']).strip())
                dataFlag = True
            else:
                dataFlag = False
            if dataFlag and dataItem['count'] is not None and str(dataItem['count']).strip() != '':
                cnt = float(str(dataItem['count']).strip())
            if isinstance(denominatorData, list):
                denominatorText = denominatorData[0]
                denominatorValue = denominatorData[1]
                denominator = 0
                for denominatorIndex, denominatorItem in enumerate(denominatorText):
                    if denominatorItem == text:
                        denominator = denominatorValue[denominatorIndex]
                        break
                if denominator > 0:
                    passRate = cnt / denominator
            if dataFlag and text is not None:
                validN += cnt
                sumTotal += sumData
                if rowIDValuesFlag:
                    if text in json.loads(rowIDValues).keys():
                        jsonValue["detailValid"].append(
                            dict(valueRow=int(float(text)), textRow=json.loads(rowIDValues)[text], sum=sumData,
                                 count=cnt,average=avg, max=maxData, min=minData, validpercent=0.0, passRate=passRate,
                                 median=0, mode=0,stdev=0,Percent5=0, Percent10=0, Percent25=0, Percent75=0,Percent90=0,
                                 Percent95=0, variance=0))
                else:
                    valueID, textRow = "0", text
                    for key, value in json.loads(rowIDValues).items():
                        if idText != "":
                            if key + value == text:
                                valueID,textRow = key, value
                                break
                        else:
                            if value == text:
                                valueID = key
                                break
                    jsonValue["detailValid"].append(
                        dict(valueRow=int(float(valueID)), textRow=textRow, sum=sumData, count=cnt, average=avg, max=maxData,
                             min=minData, validpercent=0.0, passRate=passRate))
        if sumTotal != 0:
            for detailItem in jsonValue["detailValid"]:
                detailItem["validpercent"] = detailItem["sum"] / sumTotal
        jsonValue["detailValid"].sort(key=lambda x:x["valueRow"], reverse=False)
        if str(deleteRow) != "" and isinstance(deleteRow, dict):
            detailRemove = []
            for index, detailItem in enumerate(jsonValue["detailValid"]):
                removeFlag = False
                for key, value in deleteRow.items():
                    if detailItem[key] < value:
                        removeFlag = True
                if removeFlag:
                    detailRemove.append(detailItem)
            for index, detailItem in enumerate(detailRemove):
                jsonValue["detailValid"].remove(detailItem)
        totalN = validN
        if totalN != 0:
            validP = validN / totalN
            invalidP = (totalN - validN) / totalN
        jsonValue["head"].append(dict(rowVarLabel=rowVarLabel, columnVarLabel=columnVarLabel,RowID=RowIDStr,
                                      ColumnID=VariableItem,count=int(validN),totalN=int(totalN),validN=int(validN),
                                      length=len(jsonValue["detailValid"]),variance_text='', variance_F=0, variance_P=0,
                                      validP=validP,invalidN=int(totalN - validN),invalidP=invalidP,
                                      analysis="comparemeans"))
        if percentileRank is not None:
            Condition2 = percentileRank['percentileRank']['condition']
            Condition2[0].append(RowID)
            Condition2[1].append('')
            for detailValidIndex, detailValidItem in enumerate(jsonValue["detailValid"]):
                if rowIDValuesFlag:
                    Condition2[1][len(Condition2[1]) - 1] = str(detailValidItem['valueRow'])
                else:
                    Condition2[1][len(Condition2[1]) - 1] = str(detailValidItem['textRow'])
                DataSum2 = retCompareMeans.SumMongoCompareAvg(dataSetFlag, ProjID, dataSetId, None,
                                                              percentileRank['percentileRank']['RowID'],
                                                              VariableIDgroup, optionIDgroup, "", Condition2)
                datasource2 = dataSourceSet(DataSum2, percentileRank['percentileRank']['RowID'], VariableIDgroup, '0', True)
                jsonValue["detailValid"][detailValidIndex]['percentileRank'] = \
                    getPercentileRank(detailValidItem, datasource2['detailValid'])
        if formula != '': calculation_formula(jsonValue["detailValid"], formula)
        return jsonValue
    except Exception as ex:
        return {}


# -----<排序题：排序题分析>----- #
def mongo_sort_logic(PlatID, mongodb_cfg,ProjID,dataSetId,VariableID1,VariableID2,VariableID3,point,dataSetIdFlag,
                     CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion=dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID,dataSetId)
        #retVersion.close()
    versionNum = None

    retMeta = select_meta_variable(mongodb_cfg)
    DataRowID1 = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum,VariableID1)
    DataRowID2 = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, VariableID2)
    DataRowID3 = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, VariableID3)
    #retMeta.close()
    try:
        retSort = mongoSortModel(mongodb_cfg)
        DataCount = retSort.SortMongo(dataSetFlag,ProjID,dataSetId,versionNum,VariableID1,VariableID2,VariableID3,
                                      CasesCondition)
        #retSort.close()
        varLabel1,varLabel2,varLabel3,varValues,option = "","","","",[]
        #np.seterr(divide='ignore', invalid='ignore')
        for rowItem in DataRowID1 :
            if 'varValues' in rowItem: varValues = rowItem['varValues']
            if 'varLabel' in rowItem: varLabel1 = rowItem['varLabel']
        for rowItem in DataRowID2 :
            if 'varLabel' in rowItem: varLabel2 = rowItem['varLabel']
        for rowItem in DataRowID3 :
            if 'varLabel' in rowItem: varLabel3 = rowItem['varLabel']
        retFrequency = mongoFrequencyModel(mongodb_cfg)
        DataOption = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID1, "", CasesCondition)
        for j in DataOption:
            if j['_id'] is not None and str(j['_id']).strip() != '' and str(j['_id']).strip() != 'None' and \
                    str(j['_id']).strip()!= 'null':
                option.append(j['_id'])
        DataOption = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID2, "", CasesCondition)
        for j in DataOption:
            if j['_id'] is not None and str(j['_id']).strip() != '' and str(j['_id']).strip() != 'None' and \
                    str(j['_id']).strip()!= 'null' and str(j['_id']).strip() not in option:
                option.append(j['_id'])
        DataOption = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID3, "", CasesCondition)
        for j in DataOption:
            if j['_id'] is not None and str(j['_id']).strip() != ''  and str(j['_id']).strip() != 'None' and \
                    str(j['_id']).strip()!= 'null' and str(j['_id']).strip() not in option:
                option.append(j['_id'])
        jsonValue = dict(head=[], detailValid=[])
        lists=np.empty(shape=[0,len(option)],dtype=int)
        allCount,validCount,validP,missingP = 0.0,0.0,0.0,0.0
        for dataItem in DataCount:
            allCount += float(dataItem['count'])
            if VariableID1 in dataItem['_id'].keys() or VariableID2 in dataItem['_id'].keys() or \
                VariableID3 in dataItem['_id'].keys():
                validCount += float(dataItem['count'])
                for m in range(int(dataItem['count'])):
                    counts = []
                    for n in range(len(option)):
                        if str(option[n])==str(dataItem['_id'].get(VariableID1)).strip() or \
                            str(option[n])==str(dataItem['_id'].get(VariableID2)).strip() or \
                                str(option[n])==str(dataItem['_id'].get(VariableID3)).strip():
                            if str(option[n]) == str(dataItem['_id'].get(VariableID1)).strip():
                                counts.append(5)
                            elif str(option[n]) == str(dataItem['_id'].get(VariableID2)).strip():
                                counts.append(3)
                            else:
                                counts.append(1)
                        else:
                            counts.append(0)
                    lists = np.append(lists, [counts], axis=0)
        for n in range(len(option)):
            DataSort, DataSort2 = [], []
            for m in range(int(validCount)):
                DataSort.append(float(lists[m][n]))
                if float(lists[m][n]) != 0:DataSort2.append(float(lists[m][n]))
            std = np.std(DataSort, axis=0, ddof=1)                       # 数据只有1条会报错，除0错误，返回值nan
            maxData = np.max(DataSort)
            cnt = len(DataSort2)
            sumData = np.sum(DataSort)
            minData = np.min(DataSort)
            avg = np.average(DataSort)
            var = np.var(DataSort)
            mid = np.median(DataSort)
            counts = np.bincount(DataSort)
            if cnt > 0:
                if varValues == '' or check_json_format(varValues) == False:
                    if is_number(option[n]):
                        index = int(float(option[n]))
                    else:
                        index = n
                    jsonValue["detailValid"].append(
                        dict(value=index, text=option[n], sum=sumData, count=cnt, average=avg, max=maxData,
                             median=mid, min=minData, stdev=std, variance=var))
                else:
                    optionNM=''
                    for key, value in json.loads(varValues).items():
                        if float(key) == float(option[n]):
                            optionNM = value
                    jsonValue["detailValid"].append(
                        dict(value=int(float(option[n])), text=optionNM, sum=sumData, count=cnt, average=avg, max=maxData,
                             median=mid, min=minData, stdev=std, variance=var))
        jsonValue["detailValid"].sort(key=lambda x: x["average"], reverse=True)
        sortIndex = 0
        for detailIndex, detailItem in enumerate(jsonValue["detailValid"]):
            if detailIndex == 0:
                sortIndex += 1
            else:
                if point == '':
                    if detailItem['average'] != jsonValue["detailValid"][detailIndex - 1]['average']:
                        sortIndex += 1
                else:
                    if formatdata(detailItem['average'], int(point)) != \
                        formatdata(jsonValue["detailValid"][detailIndex - 1]['average'], int(point)):
                        sortIndex += 1
            detailItem["sort"] = sortIndex
        jsonValue["detailValid"].sort(key=lambda x: x["value"], reverse=False)
        if float(allCount) != 0:
            validP = float(validCount) / float(allCount)
            missingP = float(allCount - validCount) / float(allCount)
        jsonValue["head"].append(
            dict(varLabel1=varLabel1, varLabel2=varLabel2,varLabel3=varLabel3, variableID1=VariableID1,
                 variableID2=VariableID2,variableID3=VariableID3,count=int(validCount),totalN=int(allCount),
                 validN=int(validCount),validP=validP,invalidN=int(allCount - validCount),invalidP=missingP,
                 analysis="sort"))
        return jsonValue
    except Exception as ex:
        return {}


def mongo_sort2_logic(PlatID, mongodb_cfg,ProjID,dataSetId,VariableID,phrase,point,optionPoint,dataSetIdFlag,
                      CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID,ProjID,dataSetId)
        #retVersion.close()
    versionNum = None

    try:
        retMeta = select_meta_variable(mongodb_cfg)
        DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum,VariableID)
        #retMeta.close()
        varLabel,varValues,option,optionValueLength = "","",[],3
        for rowItem in DataRowID :
            if 'varValues' in rowItem: varValues = rowItem['varValues']
            if 'varLabel' in rowItem: varLabel = rowItem['varLabel']
        if str(phrase) != "":
            varValues = json.dumps(phrase)
        retFrequency = mongoFrequencyModel(mongodb_cfg)
        DataOption = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID, "", CasesCondition)
        #retFrequency.close()
        DataSource = pd.DataFrame(list(DataOption)).values.tolist()
        if isinstance(DataSource, list) and len(DataSource) > 0:
            for j in DataSource:
                if j[0] is not None and str(j[0]).strip() != '':
                    if "," in j[0]:
                        varstr = str(j[0]).strip().split(",")
                    elif "." in j[0]:
                        varstr = str(j[0]).strip().split(".")
                    else:
                        varstr = [str(j[0]).strip()]
                    for varItem in varstr:
                        if str(varItem).strip() not in option:
                            option.append(str(varItem).strip())
                    if len(varstr) > optionValueLength:optionValueLength = len(varstr)
        if isinstance(optionPoint, list):
            optionValue = optionPoint
        else:
            if optionValueLength > 5:
                optionValue = [10,9,8,7,6,5,4,3,2,1]
            elif optionValueLength > 3:
                optionValue = [5,4,3,2,1]
            else:
                optionValue = [5,3,1]
        optionSort, optionSortSum = [], []
        for optionSortIndex in range(len(optionValue)):
            optionSortDict = {}
            for optionIndex, optionItem in enumerate(option):
                optionSortDict[optionItem] = 0.0
            optionSort.append(optionSortDict)
            optionSortSum.append(0.0)
        jsonValue = dict(head=[], detailValid=[])
        lists=np.empty(shape=[0,len(option)],dtype=int)
        allCount,validCount,validP,missingP = 0.0,0.0,0.0,0.0
        if isinstance(DataSource, list) and len(DataSource) > 0:
            for dataItem in DataSource:
                allCount += float(dataItem[1])
                if dataItem[0] is not None and str(dataItem[0]).strip() != '':
                    validCount += float(dataItem[1])
                    if "," in dataItem[0]:
                        varstr = str(dataItem[0]).strip().split(",")
                    elif "." in dataItem[0]:
                        varstr = str(dataItem[0]).strip().split(".")
                    else:
                        varstr = [str(dataItem[0]).strip()]
                    for m in range(int(float(dataItem[1]))):
                        counts = []
                        for n in range(len(option)):
                            if len(varstr) > 0 and str(option[n]).strip() in varstr:
                                counts.append(int(optionValue[varstr.index(str(option[n]).strip())]))
                            else:
                                counts.append(0)
                        lists = np.append(lists, [counts], axis=0)
                    for varstrIndex, varstrItem in enumerate(varstr):
                        for key, value in optionSort[varstrIndex].items():
                            if key == varstrItem:
                                optionSort[varstrIndex][key] = float(dataItem[1]) + value
                                break
                        optionSortSum[varstrIndex] += float(dataItem[1])
        for n in range(len(option)):
            DataSort, DataSort2, countSort, percentSort = [], [], [0] * len(optionSortSum), \
                                                          [0.0] * len(optionSortSum)
            for m in range(int(float(validCount))):
                DataSort.append(float(lists[m][n]))
                if float(lists[m][n]) != 0: DataSort2.append(float(lists[m][n]))
            std = np.std(DataSort, axis=0, ddof=1)      # 数据只有1条会报错，除0错误，返回值nan
            maxData = np.max(DataSort)
            cnt = len(DataSort2)
            sumData = np.sum(DataSort)
            minData = np.min(DataSort)
            avg = np.average(DataSort)
            var = np.var(DataSort)
            mid = np.median(DataSort)
            counts = np.bincount(DataSort)
            for optionSortSumIndex, optionSortSumItem in enumerate(optionSortSum):
                if optionSortSumItem != 0:
                    for key, value in optionSort[optionSortSumIndex].items():
                        if key == option[n]:
                            countSort[optionSortSumIndex] = int(value)
                            percentSort[optionSortSumIndex] = value / optionSortSumItem
                            break
            if cnt > 0:
                if varValues == '' or check_json_format(varValues) == False:
                    if is_number(option[n]):
                        index = int(float(option[n]))
                    else:
                        index = n
                    jsonValue["detailValid"].append(
                        dict(value=index, text=option[n], sum=sumData, count=cnt, average=avg, max=maxData,
                             median=mid, min=minData, stdev=std, variance=var, sort=0, countSort=countSort,
                             percentSort=percentSort))
                else:
                    optionNM=''
                    for key, value in json.loads(varValues).items():
                        if float(key) == float(option[n]):
                            optionNM = value
                            break
                    jsonValue["detailValid"].append(
                        dict(value=int(float(option[n])), text=optionNM, sum=sumData, count=cnt, average=avg, max=maxData,
                             median=mid, min=minData, stdev=std, variance=var, sort=0, countSort=countSort,
                             percentSort=percentSort))
        jsonValue["detailValid"].sort(key=lambda x: x["average"], reverse=True)
        sortIndex = 0
        for detailIndex, detailItem in enumerate(jsonValue["detailValid"]):
            if detailIndex == 0:
                sortIndex += 1
            else:
                if point == '':
                    if detailItem['average'] != jsonValue["detailValid"][detailIndex - 1]['average']:
                        sortIndex += 1
                else:
                    if formatdata(detailItem['average'], int(point)) != \
                        formatdata(jsonValue["detailValid"][detailIndex - 1]['average'], int(point)):
                        sortIndex += 1
            detailItem["sort"] = sortIndex
        jsonValue["detailValid"].sort(key=lambda x: x["value"], reverse=False)
        if float(allCount) != 0:
            validP = float(validCount) / float(allCount)
            missingP = float(allCount - validCount) / float(allCount)
        jsonValue["head"].append(
            dict(varLabel=varLabel, variableID=VariableID,count=int(validCount),totalN=int(allCount),
                 validN=int(validCount),validP=validP,invalidN=int(allCount - validCount),invalidP=missingP,analysis="sort"))
        #print(jsonValue["head"])
        #print(jsonValue["detailValid"])
        return jsonValue
    except Exception as ex:
        return {}


# -----<交叉表：排序题分析>----- #
def mongo_crosstabsSort2_logic(PlatID, mongodb_cfg,ProjID,dataSetId, RowID,VariableID,phrase,point,optionPoint,
                               dataSetIdFlag, deleteRow, idText, CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID,ProjID,dataSetId)
        #retVersion.close()
    versionNum = None

    try:
        retMeta = select_meta_variable(mongodb_cfg)
        DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum,RowID)
        #retMeta.close()
        rowVarLabel, rowIDValues = "", None
        for rowItem in DataRowID:
            if 'varValues' in rowItem: rowIDValues = rowItem['varValues']
            if 'varLabel' in rowItem: rowVarLabel = rowItem['varLabel']
        if not rowIDValues or str(rowIDValues).strip() == "" or not check_json_format(str(rowIDValues).strip()):
            retFrequency = mongoFrequencyModel(mongodb_cfg)
            RowData = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, "", CasesCondition)
            # retFrequency.close()
            index, rowIDValuesJson1, rowIDValuesJson2, numFlag, indexText, valueText, sortText = 0, [], [], True, "", "", []
            if idText != "": index = 999999000
            for dataItem in RowData:
                if dataItem['_id'] is not None and str(dataItem['_id']).strip() != '' and \
                        str(dataItem['_id']).strip() != 'None' and str(dataItem['_id']).strip() != 'null':
                    if idText != "" and "".join(list(filter(str.isdigit, str(dataItem['_id']).strip()))) != "":
                        indexText = "".join(list(filter(str.isdigit, str(dataItem['_id']).strip())))
                        if indexText == str(dataItem['_id']).strip():
                            valueText = str(dataItem['_id']).strip()
                        else:
                            valueText = str(dataItem['_id']).strip().replace(indexText, '')
                        if str(dataItem['_id']).strip().find(indexText) < 0:
                            charEnd = -1
                            if str(dataItem['_id']).strip().find(indexText[0:1]) == 0:
                                for charIndex in range(len(indexText)):
                                    if str(dataItem['_id']).strip().find(indexText[0:charIndex + 1]) != 0:
                                        charEnd = charIndex
                                        break
                            if charEnd != -1:
                                indexText = indexText[0:charEnd]
                                valueText = str(dataItem['_id']).strip()[charEnd:len(str(dataItem['_id']).strip())]
                            else:
                                indexText = ''
                                valueText = str(dataItem['_id']).strip()
                        if indexText == "": indexText = str(index)
                        rowIDValuesJson1.append({indexText: valueText})
                    else:
                        rowIDValuesJson1.append({str(index + 1): str(dataItem['_id']).strip()})
                    rowIDValuesJson2.append({str(dataItem['_id']).strip(): str(dataItem['_id']).strip()})
                    if not is_number(str(dataItem['_id']).strip()):
                        numFlag = False
                    if idText == "": sortText.append(str(dataItem['_id']).strip())
                    index += 1
            if numFlag:
                rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson2)))
            else:
                if len(sortText) > 0:
                    sortTextArray = sorted(sortText, key=lazy_pinyin)
                    rowIDValues = json.dumps({str(index + 1): value for index, value in enumerate(sortTextArray)})
                else:
                    rowIDValues = json.dumps(dict(ChainMap(*rowIDValuesJson1)))

        DataRowID = retMeta.SelectMetaVariable(PlatID, ProjID, dataSetId, versionNum, VariableID)
        varLabel,varValues,option,optionValueLength = "","",[],3
        for rowItem in DataRowID :
            if 'varValues' in rowItem: varValues = rowItem['varValues']
            if 'varLabel' in rowItem: varLabel = rowItem['varLabel']
        if str(phrase) != "":
            varValues = json.dumps(phrase)

        retFrequency = mongoFrequencyModel(mongodb_cfg)
        DataOption = retFrequency.CountMongo(dataSetFlag, ProjID, dataSetId, versionNum, VariableID, "", CasesCondition)
        #retFrequency.close()
        DataSource = pd.DataFrame(list(DataOption)).values.tolist()
        if isinstance(DataSource, list) and len(DataSource) > 0:
            for j in DataSource:
                if j[0] is not None and str(j[0]).strip() != '':
                    if "," in j[0]:
                        varstr = str(j[0]).strip().split(",")
                    elif "." in j[0]:
                        varstr = str(j[0]).strip().split(".")
                    else:
                        varstr = [str(j[0]).strip()]
                    for varItem in varstr:
                        if str(varItem).strip() not in option:
                            option.append(str(varItem).strip())
                    if len(varstr) > optionValueLength:optionValueLength = len(varstr)
        if isinstance(optionPoint, list):
            optionValue = optionPoint
        else:
            if optionValueLength > 5:
                optionValue = [10,9,8,7,6,5,4,3,2,1]
            elif optionValueLength > 3:
                optionValue = [5,4,3,2,1]
            else:
                optionValue = [5,3,1]

        jsonValue, lists, optionSortRow, optionSortSumRow, validRowCount  = dict(head=[], detailValid=[]), {}, {}, {},{}
        for key, value in json.loads(rowIDValues).items():
            lists[key] = np.empty(shape=[0,len(option)],dtype=int)
            optionSort, optionSortSum = [], []
            for optionSortIndex in range(len(optionValue)):
                optionSortDict = {}
                for optionIndex, optionItem in enumerate(option):
                    optionSortDict[optionItem] = 0.0
                optionSort.append(optionSortDict)
                optionSortSum.append(0.0)
            optionSortRow[key] = optionSort
            optionSortSumRow[key] = optionSortSum
            validRowCount[key] = 0.0
        allCount,validCount,validP,missingP = 0.0,0.0,0.0,0.0
        retCrosstabs = mongoCrosstabsModel(mongodb_cfg)
        DataCount = retCrosstabs.CrosstabsCountMongo(dataSetFlag, ProjID, dataSetId, versionNum, RowID, VariableID, "",
                                                     CasesCondition)
        for dataItem in DataCount:
            for key, value in json.loads(rowIDValues).items():
                if RowID in dataItem['_id'].keys() and \
                        (str(key).strip() == str(dataItem['_id'][RowID]).strip() or
                         str(value).strip() == str(dataItem['_id'][RowID]).strip() or
                         (str(key).strip() + str(value).strip() == str(dataItem['_id'][RowID]).strip() and
                          str(key).strip() != str(value).strip())):
                    allCount += float(dataItem['count'])
                    if dataItem['_id'][VariableID] is not None and str(dataItem['_id'][VariableID]).strip() != '':
                        validCount += float(dataItem['count'])
                        validRowCount[key] += float(dataItem['count'])
                        if "," in dataItem['_id'][VariableID]:
                            varstr = str(dataItem['_id'][VariableID]).strip().split(",")
                        elif "." in dataItem['_id']:
                            varstr = str(dataItem['_id'][VariableID]).strip().split(".")
                        else:
                            varstr = [str(dataItem['_id'][VariableID]).strip()]
                        for m in range(int(float(dataItem['count']))):
                            counts = []
                            for n in range(len(option)):
                                if len(varstr) > 0 and str(option[n]).strip() in varstr:
                                    counts.append(int(optionValue[varstr.index(str(option[n]).strip())]))
                                else:
                                    counts.append(0)
                            lists[key] = np.append(lists[key], [counts], axis=0)
                        for varstrIndex, varstrItem in enumerate(varstr):
                            for optionSortKey, optionSortValue in optionSortRow[key][varstrIndex].items():
                                if optionSortKey == varstrItem:
                                    optionSortRow[key][varstrIndex][optionSortKey] = \
                                        float(dataItem['count']) + optionSortValue
                                    break
                            optionSortSumRow[key][varstrIndex] += float(dataItem['count'])

        for key, value in json.loads(rowIDValues).items():
            for n in range(len(option)):
                DataSort, DataSort2, countSort, percentSort = [], [], [0] * len(optionSortSumRow[key]), \
                                                              [0.0] * len(optionSortSumRow[key])
                for m in range(int(float(validRowCount[key]))):
                    DataSort.append(float(lists[key][m][n]))
                    if float(lists[key][m][n]) != 0: DataSort2.append(float(lists[key][m][n]))
                std = np.std(DataSort, axis=0, ddof=1)      # 数据只有1条会报错，除0错误，返回值nan
                maxData = np.max(DataSort)
                cnt = len(DataSort2)
                sumData = np.sum(DataSort)
                minData = np.min(DataSort)
                avg = np.average(DataSort)
                var = np.var(DataSort)
                mid = np.median(DataSort)
                counts = np.bincount(DataSort)
                for optionSortSumIndex, optionSortSumItem in enumerate(optionSortSumRow[key]):
                    if optionSortSumItem != 0:
                        for optionKey, optionValue in optionSortRow[key][optionSortSumIndex].items():
                            if optionKey == option[n]:
                                countSort[optionSortSumIndex] = int(optionValue)
                                percentSort[optionSortSumIndex] = optionValue / optionSortSumItem
                                break
                if cnt > 0:
                    if varValues == '' or check_json_format(varValues) == False:
                        if is_number(option[n]):
                            index = int(float(option[n]))
                        else:
                            index = n
                        jsonValue["detailValid"].append(
                            dict(valueRow=int(float(key)), textRow=value,valueColumn=index, textColumn=option[n],
                                 sum=sumData, count=cnt, average=avg, max=maxData,
                                 median=mid, min=minData, stdev=std, variance=var, sort=0, countSort=countSort,
                                 total=validRowCount[key],percentSort=percentSort))
                    else:
                        optionNM=''
                        for varKey, varValue in json.loads(varValues).items():
                            if float(varKey) == float(option[n]):
                                optionNM = varValue
                                break
                        jsonValue["detailValid"].append(
                            dict(valueRow=int(float(key)), textRow=value, valueColumn=int(float(option[n])),
                                 textColumn=optionNM, sum=sumData, count=cnt, average=avg, max=maxData,
                                 median=mid, min=minData, stdev=std, variance=var, sort=0, countSort=countSort,
                                 total=validRowCount[key],percentSort=percentSort))
        if str(deleteRow) != "" and isinstance(deleteRow, dict):
            detailRemove = []
            for index, detailItem in enumerate(jsonValue["detailValid"]):
                removeFlag = False
                for key, value in deleteRow.items():
                    if detailItem[key] < value:
                        removeFlag = True
                if removeFlag:
                    detailRemove.append(detailItem)
            for index, detailItem in enumerate(detailRemove):
                jsonValue["detailValid"].remove(detailItem)
        for key, value in json.loads(rowIDValues).items():
            detailValidSort = []
            for detailIndex, detailItem in enumerate(jsonValue["detailValid"]):
                if detailItem['valueRow'] == int(float(key)):
                    detailItem['detailIndex'] = detailIndex
                    detailValidSort.append(detailItem)
            sortIndex = 0
            detailValidSort.sort(key=lambda x: x["average"], reverse=True)
            for detailIndex, detailItem in enumerate(detailValidSort):
                if detailIndex == 0:
                    sortIndex += 1
                else:
                    if point == '':
                        if detailItem['average'] != detailValidSort[detailIndex - 1]['average']:
                            sortIndex += 1
                    else:
                        if formatdata(detailItem['average'], int(point)) != \
                            formatdata(detailValidSort[detailIndex - 1]['average'], int(point)):
                            sortIndex += 1
                jsonValue["detailValid"][detailItem['detailIndex']]["sort"] = sortIndex
        jsonValue["detailValid"].sort(key=lambda x:(x["valueRow"], x["valueColumn"]), reverse=False)
        if float(allCount) != 0:
            validP = float(validCount) / float(allCount)
            missingP = float(allCount - validCount) / float(allCount)
        jsonValue["head"].append(
            dict(rowVarLabel=rowVarLabel, columnVarLabel=varLabel, RowID=RowID, ColumnID=VariableID,count=int(validCount),
                 totalN=int(allCount),validN=int(validCount),validP=validP,invalidN=int(allCount - validCount),
                 invalidP=missingP, analysis="crosstabsSort"))
        #print(jsonValue["head"])
        #print(jsonValue["detailValid"])
        return jsonValue
    except Exception as ex:
        return {}


# -----<百分等级：组合分析>----- #
def mongo_percentileRank_logic(PlatID, mongodb_cfg, ProjID,dataSetId1,dataSetId2,RowID,ColumnID1,ColumnID2,variableGroup,
                               phrase,phraseText, fullMarks,divide, dataSetIdFlag, CasesCondition1,CasesCondition2,
                               CasesCondition3):
    optionGroup,jsonValue,validN, fullMarksList, variableGroupCase, retMeans = \
        "", dict(head=[],detailValid=[]),0, fullMarks, "", mongoMeansModel(mongodb_cfg)
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId2 in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag2 = dataSetIdFlag['dataSetId'][dataSetId2]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion, dataSetFlag2 = retVersion.version(PlatID, ProjID, dataSetId2)
        #retVersion.close()
    for phraseIndex, phraseItem in enumerate(phrase):
        keys,values,Condition2,Condition3 = \
            [ColumnID1],[phraseItem],copy.deepcopy(CasesCondition2),copy.deepcopy(CasesCondition3)
        if  CasesCondition1 is None:
            variableGroupCase = [ColumnID2]
        else:
            if isinstance(CasesCondition1, str):
                if str(CasesCondition1).strip() != "":
                    values = [{str(CasesCondition1).strip().replace("regex","$regex"):phraseItem}]
                Condition1 = [keys, values]
            else:
                for keyIndex, key in enumerate(CasesCondition1[0]):
                    value = CasesCondition1[1][keyIndex]
                    keys.append(key)
                    values.append(value)
                Condition1 = [keys, values]
            if isinstance(ColumnID2, list):
                datasource = mongo_crosstabs_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2[0], ColumnID2[1],
                                                   None, "", "", "", dataSetIdFlag, Condition1)
            else:
                datasource = mongo_frequency_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2, "", "", "", "",
                                                   "", dataSetIdFlag, Condition1)
            if isinstance(variableGroup, str) and variableGroup == "":
                variableGroupCase = []
                if isinstance(ColumnID2, list):
                    phraseGroup = []
                    for detailValidItem in datasource["detailValid"]:
                        variableGroupCase.append(detailValidItem['textColumn'])
                        phraseGroup.append(detailValidItem['textRow'])
                    for keyIndex, key in enumerate(Condition2[0]):
                        value = Condition2[1][keyIndex]
                        if isinstance(value, str) and value == "phrase" and len(phraseGroup) > 0:
                            if len(phraseGroup) > 1:
                                Condition2[1][keyIndex] = {"$in" : phraseGroup}
                            else:
                                Condition2[1][keyIndex]  = phraseGroup[0]
                    for keyIndex, key in enumerate(Condition3[0]):
                        value = Condition3[1][keyIndex]
                        if isinstance(value, str) and value == "phrase" and len(phraseGroup) > 0:
                            if len(phraseGroup) > 1:
                                Condition3[1][keyIndex] = {"$in" : phraseGroup}
                            else:
                                Condition3[1][keyIndex] = phraseGroup[0]
                else:
                    for detailValidItem in datasource["detailValid"]:
                        if isinstance(detailValidItem['text'], list):
                            textMore = ""
                            for textItem in detailValidItem['text']:
                                if textMore != "" : textMore += "*"
                                textMore += textItem
                            variableGroupCase.append(textMore)
                        else:
                            variableGroupCase.append(detailValidItem['text'])
                    for keyIndex, key in enumerate(Condition2[0]):
                        value = Condition2[1][keyIndex]
                        if isinstance(value, str) and value == "phrase":
                            Condition2[1][keyIndex] = phraseItem
                    for keyIndex, key in enumerate(Condition3[0]):
                        value = Condition3[1][keyIndex]
                        if isinstance(value, str) and value == "phrase":
                            Condition3[1][keyIndex] = phraseItem
            else:
                optionGroup, variableGroupCase = [], variableGroup
                for detailValidItem in datasource["detailValid"]:
                    optionGroup.append(detailValidItem['text'])
                fullMarks = str(len(optionGroup))
            if isinstance(fullMarksList, list):
                for detailValidItem in fullMarksList:
                    if str(detailValidItem["textRow"]).strip() == str(phraseItem).strip():
                        fullMarks = str(detailValidItem["sum"])
        if len(variableGroupCase) > 0:
            if divide != "":
                DataSum1 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, fullMarks, "", Condition2)
                DataSum2 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, fullMarks, "", Condition3)
                DataSum3 = retMeans.SumMongoCompareAvg(dataSetFlag2, ProjID, dataSetId2, None, RowID,
                                                       variableGroupCase, optionGroup, "",Condition3)
            else:
                DataSum1 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, divide, "", Condition2)
                DataSum2 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, divide, "", Condition3)
                DataSum3 = retMeans.SumMongoCompareAvg(dataSetFlag2, ProjID, dataSetId2, None, RowID,
                                                       variableGroupCase, optionGroup, "",Condition3)
            datasource1 = dataSourceSet(DataSum1, RowID, variableGroupCase, fullMarks)
            datasource2 = dataSourceSet(DataSum2, RowID, variableGroupCase, fullMarks)
            datasource3 = dataSourceSet(DataSum3, RowID, variableGroupCase, fullMarks, True)
        else:
            datasource1, datasource2, datasource3 = None, None, None
        textRow = phraseItem
        if isinstance(phraseText, list) and phraseIndex < len(phraseText):
            textRow = phraseText[phraseIndex]
        if datasource1 is not None and len(datasource1['head']) > 0 \
                and len(datasource2['head']) > 0 and len(datasource3['detailValid']) > 0:
            validN += datasource2['head'][0]['count']
            if divide == "":
                if datasource1['head'][0]['masteryRate'] > 0:
                    jsonValue["detailValid"].append(
                        dict(valueRow=float(phraseIndex + 1), textRow=textRow,average=datasource1['head'][0]['average'],
                             masteryRate=datasource1['head'][0]['masteryRate'],
                             masteryRateAll=datasource2['head'][0]['masteryRate'],
                             count=datasource2['head'][0]['count'],
                             percentileRank=getPercentileRank(datasource1['head'][0], datasource3['detailValid'])))
            else:
                if datasource1['head'][0]['average'] > 0:
                    jsonValue["detailValid"].append(
                        dict(valueRow=float(phraseIndex + 1), textRow=textRow,average=datasource1['head'][0]['average'],
                             masteryRate=datasource1['head'][0]['average'],
                             masteryRateAll=datasource2['head'][0]['average'],
                             count=datasource2['head'][0]['count'],
                             percentileRank=getPercentileRank(datasource1['head'][0], datasource3['detailValid'])))
    jsonValue["head"].append(
        dict(rowVarLabel="", columnVarLabel="", RowID=RowID, ColumnID=ColumnID1, count=int(validN), totalN=int(validN),
             validN=int(validN), analysis="comparemeans"))
    #retMeans.close()
    return jsonValue


def mongo_percentileRank2_logic(PlatID, mongodb_cfg, ProjID,dataSetId1,dataSetId2,RowID,ColumnID1,ColumnID2,variableGroup,
                                phrase,phrase2, phraseText,fullMarks,divide,dataSetIdFlag, CasesCondition1,
                                CasesCondition2, CasesCondition3):
    optionGroup,jsonValue,validAll, fullMarksList, variableGroupCase, fullMarksData,retMeans = \
        "", dict(head=[],detailValid=[]),0, fullMarks, "", [], mongoMeansModel(mongodb_cfg)
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId2 in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag2 = dataSetIdFlag['dataSetId'][dataSetId2]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion, dataSetFlag2 = retVersion.version(PlatID, ProjID, dataSetId2)
        #retVersion.close()
    if isinstance(fullMarks, str) and str(fullMarks).split() != "":
        fullMarksData = fullMarks.split(':')
    for phraseIndex, phraseItem in enumerate(phrase):
        dataSum, dataSumAll, fullMarkCount, fullMarkCountAll, validN, VariableMore = 0, 0, 0.0, 0.0, 0, []
        for phrase2Index, phrase2Item in enumerate(phrase2):
            keys,values,Condition2,Condition3 = \
                [ColumnID1],[phraseItem],copy.deepcopy(CasesCondition2),copy.deepcopy(CasesCondition3)
            if isinstance(CasesCondition1, str):
                if str(CasesCondition1).strip() != "":
                    values = [{str(CasesCondition1).strip().replace("regex","$regex"):phraseItem}]
                Condition1 = [keys, values]
            else:
                for keyIndex, key in enumerate(CasesCondition1[0]):
                    value = CasesCondition1[1][keyIndex]
                    keys.append(key)
                    if value == "phrase2": value = phrase2Item
                    values.append(value)
                Condition1 = [keys, values]
            if isinstance(ColumnID2, list):
                datasource = mongo_crosstabs_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2[0], ColumnID2[1],
                                                   None, "", "","",dataSetIdFlag, Condition1)
            else:
                datasource = mongo_frequency_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2, "", "", "", "",
                                                   "", dataSetIdFlag, Condition1)
            if isinstance(variableGroup, str) and variableGroup == "":
                variableGroupCase,VariableMore = [],[]
                if isinstance(ColumnID2, list):
                    phraseGroup = []
                    for detailValidItem in datasource["detailValid"]:
                        variableGroupCase.append(detailValidItem['textColumn'])
                        phraseGroup.append(detailValidItem['textRow'])
                    for keyIndex, key in enumerate(Condition2[0]):
                        value = Condition2[1][keyIndex]
                        if isinstance(value, str) and value == "phrase" and len(phraseGroup) > 0:
                            if len(phraseGroup) > 1:
                                Condition2[1][keyIndex] = {"$in" : phraseGroup}
                            else:
                                Condition2[1][keyIndex] = phraseGroup[0]
                        if isinstance(value, str) and value == "phrase2":
                            Condition2[1][keyIndex] = phrase2Item
                    for keyIndex, key in enumerate(Condition3[0]):
                        value = Condition3[1][keyIndex]
                        if isinstance(value, str) and value == "phrase" and len(phraseGroup) > 0:
                            if len(phraseGroup) > 1:
                                Condition3[1][keyIndex] = {"$in" : phraseGroup}
                            else:
                                Condition3[1][keyIndex]= phraseGroup[0]
                        if isinstance(value, str) and value == "phrase2":
                            Condition3[1][keyIndex] = phrase2Item
                else:
                    for detailValidItem in datasource["detailValid"]:
                        if isinstance(detailValidItem['text'], list):
                            textMore = ""
                            for textItem in detailValidItem['text']:
                                if textMore != "": textMore += "*"
                                textMore += textItem
                            variableGroupCase.append(textMore)
                        else:
                            variableGroupCase.append(detailValidItem['text'])
                    for keyIndex, key in enumerate(Condition2[0]):
                        value = Condition2[1][keyIndex]
                        if isinstance(value, str) and value == "phrase":
                            Condition2[1][keyIndex] = phraseItem
                        if isinstance(value, str) and value == "phrase2":
                            Condition2[1][keyIndex] = phrase2Item
                    for keyIndex, key in enumerate(Condition3[0]):
                        value = Condition3[1][keyIndex]
                        if isinstance(value, str) and value == "phrase":
                            Condition3[1][keyIndex] = phraseItem
                        if isinstance(value, str) and value == "phrase2":
                            Condition3[1][keyIndex] = phrase2Item
            else:
                optionGroup, variableGroupCase = [], variableGroup
                for detailValidItem in datasource["detailValid"]:
                    optionGroup.append(detailValidItem['text'])
                fullMarks = str(len(optionGroup))
            if isinstance(fullMarks, str) and str(fullMarks).split() != "":
                datasource = mongo_comparemeans2_logic(PlatID, mongodb_cfg, ProjID, dataSetId1,ColumnID1, [fullMarksData[0]],
                                                       optionGroup, "", "", "", "", dataSetIdFlag, None, Condition1)
                fullMarksList = datasource["detailValid"]
            if isinstance(fullMarksList, list):
                for detailValidItem in fullMarksList:
                    if str(detailValidItem["textRow"]).strip() == str(phraseItem).strip():
                        fullMarks = str(detailValidItem["sum"])
            if len(variableGroupCase) > 0:
                DataSum1 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, divide, "", Condition2)
                DataSum2 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, divide, "", Condition3)
                if variableGroupCase[0].find("*") > -1:
                    VariableMore = variableGroupCase[0].split('*')
                elif variableGroupCase[0].find("/") > -1:
                    VariableMore = variableGroupCase[0].split('/')
                elif variableGroupCase[0].find("+") > -1:
                    VariableMore = variableGroupCase[0].split('+')
                elif variableGroupCase[0].find("-") > -1:
                    VariableMore = variableGroupCase[0].split('-')
                if len(VariableMore) > 0:
                    VariableItem = VariableMore[0]
                else:
                    VariableItem = variableGroupCase[0]
                for dataItem in DataSum1:
                    if dataItem[VariableItem + 'avg'] is not None and \
                            str(dataItem[VariableItem+ 'avg']).strip() != '':
                        dataSum = dataSum + float(str(dataItem[VariableItem + 'sum']).strip())
                        if dataItem['count'] is not None and str(dataItem['count']).strip() != '':
                            count = float(str(dataItem['count']).strip())
                            validN += count
                            validAll += count
                            fullMarkCount = fullMarkCount + float(fullMarks) * count
                for dataItem in DataSum2:
                    if dataItem[VariableItem + 'avg'] is not None and \
                            str(dataItem[VariableItem + 'avg']).strip() != '':
                        dataSumAll = dataSumAll + float(str(dataItem[VariableItem + 'sum']).strip())
                        if dataItem['count'] is not None and str(dataItem['count']).strip() != '':
                            count = float(str(dataItem['count']).strip())
                            fullMarkCountAll = fullMarkCountAll + float(fullMarks) * count
        textRow = phraseItem
        if isinstance(phraseText, list) and phraseIndex < len(phraseText):
            textRow = phraseText[phraseIndex]
        if dataSum/fullMarkCount > 0:
            jsonValue["detailValid"].append(
                dict(valueRow=float(phraseIndex + 1), textRow=textRow, average=dataSum/validN,
                     masteryRate=dataSum/fullMarkCount,
                     masteryRateAll=dataSumAll/fullMarkCountAll,
                     count=validN,percentileRank=0))
    jsonValue["head"].append(
        dict(rowVarLabel="", columnVarLabel="", RowID=RowID, ColumnID=ColumnID1, count=int(validAll),
             totalN=int(validAll),validN=int(validAll), analysis="comparemeans"))
    #retMeans.close()
    return jsonValue


def mongo_percentileRank3_logic(PlatID, mongodb_cfg, ProjID,dataSetId1,dataSetId2,RowID,ColumnID1,ColumnID2,
                                variableGroup,phrase,phrase2, phraseText,fullMarks,divide,dataSetIdFlag, CasesCondition1,
                                CasesCondition2,CasesCondition3):
    optionGroup,jsonValue,validAll, fullMarksList, variableGroupCase, fullMarksData,retMeans = \
        "", dict(head=[],detailValid=[]),0, fullMarks, "", [], mongoMeansModel(mongodb_cfg)
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId2 in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag2 = dataSetIdFlag['dataSetId'][dataSetId2]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion, dataSetFlag2 = retVersion.version(PlatID, ProjID, dataSetId2)
        #retVersion.close()
    for phraseIndex, phraseItem in enumerate(phrase):
        dataSum, dataSumAll, fullMarkCount, fullMarkCountAll, validN, VariableMore = 0, 0, 0, 0, 0, []
        for phrase2Index, phrase2Item in enumerate(phrase2):
            keys,values,Condition2,Condition3 = \
                [ColumnID1],[phraseItem],copy.deepcopy(CasesCondition2),copy.deepcopy(CasesCondition3)
            if isinstance(CasesCondition1, str):
                if str(CasesCondition1).strip() != "":
                    values = [{str(CasesCondition1).strip().replace("regex","$regex"):phraseItem}]
                Condition1 = [keys, values]
            else:
                for keyIndex, key in enumerate(CasesCondition1[0]):
                    value = CasesCondition1[1][keyIndex]
                    keys.append(key)
                    if value == "phrase2": value = phrase2Item
                    values.append(value)
                Condition1 = [keys, values]
            if isinstance(ColumnID2, list):
                datasource = mongo_crosstabs_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2[0], ColumnID2[1],
                                                   None, "", "","", dataSetIdFlag, Condition1)
            else:
                datasource = mongo_frequency_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2, "", "", "", "",
                                                   "", dataSetIdFlag, Condition1)
            if isinstance(variableGroup, str) and variableGroup == "":
                variableGroupCase,VariableMore = [], []
                if isinstance(ColumnID2, list):
                    phraseGroup = []
                    for detailValidItem in datasource["detailValid"]:
                        variableGroupCase.append(detailValidItem['textColumn'])
                        phraseGroup.append(detailValidItem['textRow'])
                    for keyIndex, key in enumerate(Condition2[0]):
                        value = Condition2[1][keyIndex]
                        if isinstance(value, str) and value == "phrase" and len(phraseGroup) > 0:
                            if len(phraseGroup) > 1:
                                Condition2[1][keyIndex] = {"$in" : phraseGroup}
                            else:
                                Condition2[1][keyIndex] = phraseGroup[0]
                        if isinstance(value, str) and value == "phrase2":
                            Condition2[1][keyIndex] = phrase2Item
                    for keyIndex, key in enumerate(Condition3[0]):
                        value = Condition3[1][keyIndex]
                        if isinstance(value, str) and value == "phrase" and len(phraseGroup) > 0:
                            if len(phraseGroup) > 1:
                                Condition3[1][keyIndex] = {"$in" : phraseGroup}
                            else:
                                Condition3[1][keyIndex] = phraseGroup[0]
                        if isinstance(value, str) and value == "phrase2":
                            Condition3[1][keyIndex] = phrase2Item
                else:
                    for detailValidItem in datasource["detailValid"]:
                        if isinstance(detailValidItem['text'], list):
                            textMore = ""
                            for textItem in detailValidItem['text']:
                                if textMore != "": textMore += "*"
                                textMore += textItem
                            variableGroupCase.append(textMore)
                        else:
                            variableGroupCase.append(detailValidItem['text'])
                    for keyIndex, key in enumerate(Condition2[0]):
                        value = Condition2[1][keyIndex]
                        if isinstance(value, str) and value == "phrase":
                            Condition2[1][keyIndex] = phraseItem
                        if isinstance(value, str) and value == "phrase2":
                            Condition2[1][keyIndex] = phrase2Item
                    for keyIndex, key in enumerate(Condition3[0]):
                        value = Condition3[1][keyIndex]
                        if isinstance(value, str) and value == "phrase":
                            Condition3[1][keyIndex] = phraseItem
                        if isinstance(value, str) and value == "phrase2":
                            Condition3[1][keyIndex] = phrase2Item
            else:
                optionGroup, variableGroupCase = [], variableGroup
                for detailValidItem in datasource["detailValid"]:
                    optionGroup.append(detailValidItem['text'])
                fullMarks = str(len(optionGroup))
            if isinstance(fullMarksList, list):
                for detailValidItem in fullMarksList:
                    if str(detailValidItem["textRow"]).strip() == str(phraseItem).strip() + str(phrase2Item).strip():
                        fullMarks = str(detailValidItem["sum"])
            if len(variableGroupCase) > 0:
                if divide != "":
                    DataSum1 = retMeans.SumMongo(dataSetFlag2, ProjID, dataSetId2, None, "", variableGroupCase,
                                                 fullMarks, "", Condition2)
                    DataSum2 = retMeans.SumMongo(dataSetFlag2, ProjID, dataSetId2, None, "", variableGroupCase,
                                                 fullMarks, "", Condition3)
                else:
                    DataSum1 = retMeans.SumMongo(dataSetFlag2, ProjID, dataSetId2, None, "", variableGroupCase,
                                                 divide, "", Condition2)
                    DataSum2 = retMeans.SumMongo(dataSetFlag2, ProjID, dataSetId2, None, "", variableGroupCase,
                                                 divide, "", Condition3)
                if variableGroupCase[0].find("*") > -1:
                    VariableMore = variableGroupCase[0].split('*')
                elif variableGroupCase[0].find("/") > -1:
                    VariableMore = variableGroupCase[0].split('/')
                elif variableGroupCase[0].find("+") > -1:
                    VariableMore = variableGroupCase[0].split('+')
                elif variableGroupCase[0].find("-") > -1:
                    VariableMore = variableGroupCase[0].split('-')
                if len(VariableMore) > 0:
                    VariableItem = VariableMore[0]
                else:
                    VariableItem = variableGroupCase[0]
                for dataItem in DataSum1:
                    if VariableItem + 'data' in dataItem.keys() and dataItem[VariableItem + 'data'] is not None and \
                            str(dataItem[VariableItem + 'data']).strip() != '':
                        dataSum = dataSum + float(str(dataItem[VariableItem + 'data']).strip())
                        fullMarkCount += 1
                        validN += 1
                        validAll += 1
                for dataItem in DataSum2:
                    if VariableItem + 'data' in dataItem.keys() and dataItem[VariableItem + 'data'] is not None and \
                            str(dataItem[VariableItem + 'data']).strip() != '':
                        dataSumAll = dataSumAll + float(str(dataItem[VariableItem + 'data']).strip())
                        fullMarkCountAll += 1
        textRow = phraseItem
        if isinstance(phraseText, list) and phraseIndex < len(phraseText):
            textRow = phraseText[phraseIndex]
        if fullMarkCount > 0 and dataSum / fullMarkCount > 0:
            jsonValue["detailValid"].append(
                dict(valueRow=float(phraseIndex + 1), textRow=textRow, average=dataSum/validN,
                     masteryRate=dataSum/fullMarkCount,
                     masteryRateAll=dataSumAll/fullMarkCountAll,
                     count=validN,percentileRank=0))
    jsonValue["head"].append(
        dict(rowVarLabel="", columnVarLabel="", RowID=RowID, ColumnID=ColumnID1, count=int(validAll),
             totalN=int(validAll),validN=int(validAll), analysis="comparemeans"))
    #retMeans.close()
    return jsonValue


def mongo_percentileRank4_logic(PlatID, mongodb_cfg, ProjID,dataSetId1,dataSetId2,RowID,ColumnID1,ColumnID2,
                                variableGroup,phrase,phraseText,fullMarks,divide, dataSetIdFlag,
                                CasesCondition1,CasesCondition2,CasesCondition3):
    optionGroup,jsonValue,validN, fullMarksList, variableGroupCase, retMeans = \
        "", dict(head=[],detailValid=[]),0, fullMarks, "", mongoMeansModel(mongodb_cfg)
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId2 in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag2 = dataSetIdFlag['dataSetId'][dataSetId2]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion, dataSetFlag2 = retVersion.version(PlatID, ProjID, dataSetId2)
        #retVersion.close()
    for phraseIndex, phraseItem in enumerate(phrase):
        keys,values,Condition2,Condition3 = \
            [ColumnID1],[phraseItem],copy.deepcopy(CasesCondition2),copy.deepcopy(CasesCondition3)
        if isinstance(CasesCondition1, str):
            if str(CasesCondition1).strip() != "":
                values = [{str(CasesCondition1).strip().replace("regex","$regex"):phraseItem}]
            Condition1 = [keys, values]
        else:
            for keyIndex, key in enumerate(CasesCondition1[0]):
                value = CasesCondition1[1][keyIndex]
                keys.append(key)
                values.append(value)
            Condition1 = [keys, values]
        if isinstance(ColumnID2, list):
            datasource = mongo_crosstabs_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2[0], ColumnID2[1],
                                               None, "", "", "", dataSetIdFlag, Condition1)
        else:
            datasource = mongo_frequency_logic(PlatID, mongodb_cfg, ProjID, dataSetId1, ColumnID2, "", "", "", "",
                                               "", dataSetIdFlag, Condition1)
        if isinstance(variableGroup, str) and variableGroup == "":
            variableGroupCase = []
            if isinstance(ColumnID2, list):
                phraseGroup = []
                for detailValidItem in datasource["detailValid"]:
                    variableGroupCase.append(detailValidItem['textColumn'])
                    phraseGroup.append(detailValidItem['textRow'])
                for keyIndex, key in enumerate(Condition2[0]):
                    value = Condition2[1][keyIndex]
                    if isinstance(value, str) and value == "phrase" and len(phraseGroup) > 0:
                        if len(phraseGroup) > 1:
                            Condition2[1][keyIndex] = {"$in" : phraseGroup}
                        else:
                            Condition2[1][keyIndex] = phraseGroup[0]
            else:
                for detailValidItem in datasource["detailValid"]:
                    if isinstance(detailValidItem['text'], list):
                        textMore = ""
                        for textItem in detailValidItem['text']:
                            if textMore != "" : textMore += "*"
                            textMore += textItem
                        variableGroupCase.append(textMore)
                    else:
                        variableGroupCase.append(detailValidItem['text'])
                for keyIndex, key in enumerate(Condition2[0]):
                    value = Condition2[1][keyIndex]
                    if isinstance(value, str) and value == "phrase":
                        Condition2[1][keyIndex] = phraseItem
        else:
            optionGroup, variableGroupCase = [], variableGroup
            for detailValidItem in datasource["detailValid"]:
                optionGroup.append(detailValidItem['text'])
            fullMarks = str(len(optionGroup))
        if isinstance(fullMarksList, list):
            for detailValidItem in fullMarksList:
                if str(detailValidItem["textRow"]).strip() == str(phraseItem).strip():
                    fullMarks = str(detailValidItem["sum"])
                    break
        if len(variableGroupCase) > 0:
            if divide != "":
                DataSum1 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, fullMarks, "", Condition2)

            else:
                DataSum1 = retMeans.SumMongoAvg(dataSetFlag2, ProjID, dataSetId2, None, variableGroupCase,
                                                optionGroup, divide, "", Condition2)
            datasource1 = dataSourceSet(DataSum1, RowID, variableGroupCase, fullMarks)

        else:
            datasource1 = None
        textRow = phraseItem
        if isinstance(phraseText, list) and phraseIndex < len(phraseText):
            textRow = phraseText[phraseIndex]
        if datasource1 is not None and len(datasource1['head']) > 0 :
            validN += 0
            if divide == "":
                if datasource1['head'][0]['masteryRate'] > 0:
                    jsonValue["detailValid"].append(
                        dict(valueRow=float(phraseIndex + 1), textRow=textRow,average=datasource1['head'][0]['average'],
                             masteryRate=datasource1['head'][0]['masteryRate'],
                             masteryRateAll=0,
                             count=datasource1['head'][0]['count'],
                             percentileRank=0))
            else:
                if datasource1['head'][0]['average'] > 0:
                    jsonValue["detailValid"].append(
                        dict(valueRow=float(phraseIndex + 1), textRow=textRow,average=datasource1['head'][0]['average'],
                             masteryRate=datasource1['head'][0]['average'],
                             masteryRateAll=0,
                             count=datasource1['head'][0]['count'],
                             percentileRank=0))
    jsonValue["head"].append(
        dict(rowVarLabel="", columnVarLabel="", RowID=RowID, ColumnID=ColumnID1, count=int(validN), totalN=int(validN),
             validN=int(validN), analysis="comparemeans"))
    #retMeans.close()
    return jsonValue


def dataSourceSet(DataSum, RowID, variableGroupCase, fullMarks, detailFlag=None):
    datasource =  dict(head=[], detailValid=[])
    sumData, avg, passRate, maxData, mid, minData, masteryRate, cnt, text, value, VariableMore = \
        0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, "", 0, []
    if variableGroupCase[0].find("*") > -1:
        VariableMore = variableGroupCase[0].split('*')
    elif variableGroupCase[0].find("/") > -1:
        VariableMore = variableGroupCase[0].split('/')
    elif variableGroupCase[0].find("+") > -1:
        VariableMore = variableGroupCase[0].split('+')
    elif variableGroupCase[0].find("-") > -1:
        VariableMore = variableGroupCase[0].split('-')
    if len(VariableMore) > 0:
        VariableItem = VariableMore[0]
    else:
        VariableItem = variableGroupCase[0]
    if detailFlag:
        for dataItem in DataSum:
            value += 1
            if dataItem['_id'][RowID] is not None and \
                str(dataItem['_id'][RowID]).strip() != '':
                text = str(dataItem['_id'][RowID]).strip()
            if dataItem[VariableItem + 'avg'] is not None and \
                str(dataItem[VariableItem + 'avg']).strip() != '':
                avg = float(str(dataItem[VariableItem + 'avg']).strip())
                sumData = float(str(dataItem[VariableItem + 'sum']).strip())
                maxData = float(str(dataItem[VariableItem + 'max']).strip())
                minData = float(str(dataItem[VariableItem + 'min']).strip())
            if dataItem['count'] is not None and str(dataItem['count']).strip() != '':
                cnt = int(str(dataItem['count']).strip())
            datasource["detailValid"].append(
                dict(valueRow=value, textRow=text, sum=sumData, count=cnt, average=avg, max=maxData,
                     median=mid, min=minData, passRate=passRate))
    else:
        for dataItem in DataSum:
            if dataItem[VariableItem + 'avg'] is not None and \
                    str(dataItem[VariableItem + 'avg']).strip() != '':
                avg = float(str(dataItem[VariableItem + 'avg']).strip())
                sumData = float(str(dataItem[VariableItem + 'sum']).strip())
                maxData = float(str(dataItem[VariableItem + 'max']).strip())
                minData = float(str(dataItem[VariableItem + 'min']).strip())
            if dataItem['count'] is not None and str(dataItem['count']).strip() != '':
                cnt = int(str(dataItem['count']).strip())
        if fullMarks != "" and float(fullMarks) != 0:
            masteryRate = avg / float(fullMarks)                #均值方式计算
            #masteryRate = sumData / (float(fullMarks) * cnt)   #总分方式计算
        datasource["head"].append(
            dict(varLabel="", variableID="", sum=sumData, count=cnt, average=avg, passRate=passRate,
                 max=maxData, median=mid, min=minData, masteryRate=masteryRate, analysis="means"))
    return datasource


def getPercentileRank(head, detailValid):
    average = head['average']
    moleculesCount, denominatorCount = 0, len(detailValid)
    detailValid.sort(key=lambda x: x["average"], reverse=False)
    for detailValidItem in detailValid:
        if detailValidItem['average'] <= average:
            moleculesCount += 1
        else:
            break
    if denominatorCount < 10:
        return None
    else:
        return moleculesCount / denominatorCount


# -----<线性回归>----- #
def mongoLinearRegressionDetail(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID, dataSetIdFlag,
                                CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
        #retVersion.close()
    versionNum = None

    linear_obj = mongoLinearRegressionAnalyModel(mongodb_cfg)
    linear_data = linear_obj.showDatas(dataSetFlag, ProjID, dataSetId, versionNum, RowID, ColumnID, CasesCondition)
    #linear_obj.close()
    df_dropna = DataFrame(linear_data, dtype=np.float64).dropna()
    df_standard = DataFrame()
    xVarName,yVarName = [],[]
    xVarName.append(RowID)
    if isinstance(ColumnID, list):
        yVarName = ColumnID
    else:
        yVarName.append(ColumnID)
    for i in xVarName + yVarName:
        m = df_dropna[i].mean()
        std = df_dropna[i].std()
        lambda_format = lambda x: (x - m) / std
        df_standard[i] = df_dropna[i].apply(lambda_format)

    # 一个因变量，多个自变量
    myDmatricesStr = " {} ~ {}".format(yVarName[0], " + ".join(xVarName))
    y, x = dmatrices(myDmatricesStr, data=df_dropna, return_type='dataframe')
    y_standard, x_standard = dmatrices(myDmatricesStr, data=df_standard, return_type='dataframe')

    mod = sm.OLS(y, x)
    mod_standard = sm.OLS(y_standard, x_standard)

    res = mod.fit()
    res_standard = mod_standard.fit()
    # print(list(res_standard.params))
    # print(res_standard.summary())

    model = {
        'n': int(res.nobs),
        'df': res.df_model,
        'df_resid': res.df_resid,
        'r': math.sqrt(res.rsquared),
        'r_squared': '{:.5f}'.format(res.rsquared),
        'r_squared_adj': '{:.5f}'.format(res.rsquared_adj),
        'f_statistic': '{:.5f}'.format(res.fvalue),
        'prob_f_statistic': '{:.5f}'.format(res.f_pvalue),
    }
    coefficient = {
        "standard_coeff": ['.'] + list(res_standard.params)[1:],
        'coefficient': list(res.params),
        'std': list(np.diag(np.sqrt(res.cov_params()))),
        't': list(res.tvalues),
        'sig': [float(i) for i in
                ("".join("{:.4f}," * len(res.pvalues)).format(*list(res.pvalues))).rstrip(",").split(",")]
    }
    returnValue = {'model': model, 'coefficient': coefficient}

    return returnValue


# -----<T检验>----- #
def mongoTtest_indResultDetail(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID,specifiedValue1, specifiedValue2,
                               dataSetIdFlag, CasesCondition=None):
    try:
        if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
            dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        else:
            retVersion = dataversion(mongodb_cfg)
            dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
            #retVersion.close()
        versionNum = None

        tTestObj = mongoLinearRegressionAnalyModel(mongodb_cfg)
        tTestData = tTestObj.showDatas(dataSetFlag, ProjID, dataSetId, versionNum, RowID, ColumnID, CasesCondition)
        #tTestObj.close()
        #df_dropna = DataFrame(tTestData, dtype=np.float64).dropna()
        if isinstance(specifiedValue1, str):
            df_dropna = DataFrame(tTestData).dropna()
            A = df_dropna[df_dropna[RowID] == str(specifiedValue1)][ColumnID]
            B = df_dropna[df_dropna[RowID] == str(specifiedValue2)][ColumnID]
            A = DataFrame(A, dtype=np.float64)
            B = DataFrame(B, dtype=np.float64)
        else:
            df_dropna = DataFrame(tTestData, dtype=np.float64).dropna()
            A = df_dropna[df_dropna[RowID] == float(str(specifiedValue1))][ColumnID]
            B = df_dropna[df_dropna[RowID] == float(str(specifiedValue2))][ColumnID]
        # DataSort = [A.values.tolist(), B.values.tolist()]
        F, Sig = tuple(stats.levene(A, B, center="mean"))
        EqualTtestValue, EqualTtestP = tuple(stats.ttest_ind(A, B))
        UnequalTtestValue, UnequalTtestP = tuple(stats.ttest_ind(A, B, equal_var=False))

        if isinstance(specifiedValue1, str):
            data = {
                "F": F,
                "Sig": EqualTtestP[0] if Sig[0] > 0.05 else UnequalTtestP[0],
                "t_sig": EqualTtestValue[0] if Sig[0] > 0.05 else UnequalTtestValue[0],
                "type": "方差齐性" if Sig[0] > 0.05 else "方差不齐性",
            }
        else:
            data = {
                "F": F,
                "Sig": EqualTtestP if Sig > 0.05 else UnequalTtestP,
                "t_sig": EqualTtestValue if Sig > 0.05 else UnequalTtestValue,
                "type": "方差齐性" if Sig > 0.05 else "方差不齐性",
            }
        return data
    except Exception as e:
        print(e)
        return {}


# -----<相关分析>----- #
def mongoCorrelationAnalysisDetail(PlatID, mongodb_cfg, ProjID, dataSetId, ColumnIDs, Corr, dataSetIdFlag,
                                   CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
        #retVersion.close()
    versionNum = None

    correlationObj = mongoCorrelationAnalysisModel(mongodb_cfg)
    correlationData = correlationObj.showDatas(dataSetFlag, ProjID, dataSetId, versionNum, ColumnIDs, CasesCondition)
    #correlationObj.close()
    df_dropna = DataFrame(correlationData, dtype=np.float64).dropna()
    # spearman 斯皮尔曼系数
    # kendall 肯德尔系数
    # pearson 皮尔逊系数
    # return pearsonr(df_dropna[xVariable], df_dropna[yVariable])
    # return df_dropna.corr(method=Corr).to_dict()
    data = dict()
    try:
        if len(df_dropna) > 0:
            for i in ColumnIDs:
                subdata = dict()
                for j in ColumnIDs:
                    analysis_data = dict()
                    '''
                        r : float
                            Pearson's correlation coefficient
                        p-value : float
                            2-tailed p-value
                    '''
                    if Corr is None:
                        pearson_r, pearson_p_value = stats.pearsonr(df_dropna[i], df_dropna[j])
                        spearman_r, spearman_p_value = stats.spearmanr(df_dropna[i], df_dropna[j])
                        analysis_data['pearson_r'] = pearson_r
                        analysis_data['pearson_p_value'] = pearson_p_value
                        analysis_data['spearman_r'] = spearman_r
                        analysis_data['spearman_p_value'] = spearman_p_value
                        analysis_data['count'] = int(df_dropna[j].count())
                    else:
                        if Corr == "pearson":
                            r, p_value = stats.pearsonr(df_dropna[i], df_dropna[j])
                        elif Corr == "spearman":
                            r, p_value = stats.spearmanr(df_dropna[i], df_dropna[j])
                        elif Corr == "kendall":
                            r, p_value = stats.kendalltau(df_dropna[i], df_dropna[j])
                        else:
                            r, p_value = 0.0,0.0
                        analysis_data['r'] = r
                        analysis_data['p_value'] = '{:.5f}'.format(p_value)
                        analysis_data['count'] = int(df_dropna[j].count())
                    subdata[j] = analysis_data

                data[i] = subdata
    except Exception as ex:
        return {}
    return data


# -----<二元Logistics回归分析>----- #
def mongoTwoDimensionalLogisticRegressionDetail(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, ColumnID, dataSetIdFlag,
                                                CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
        #retVersion.close()
    versionNum = None

    two_obj = mongoLinearRegressionAnalyModel(mongodb_cfg)
    twoData = two_obj.showDatas(dataSetFlag, ProjID, dataSetId, versionNum, RowID, ColumnID, CasesCondition)
    #two_obj.close()
    df_dropna = DataFrame(twoData, dtype=np.float64).dropna()
    df_X = DataFrame()
    df_Y = DataFrame()  # 因变量，0， 1

    df_X[RowID] = df_dropna[RowID]
    df_Y[ColumnID] = df_dropna[ColumnID]

    df_X["intercept"] = 1.0  # 截距项，很重要的呦，我就错在这里了

    YColumnList = list(df_Y[ColumnID].values)
    setYColumnList = list(set(YColumnList))

    if len(setYColumnList) > 2 or len(setYColumnList) < 2:
        raise MyCustomError(retcode=4015)
    # else:
    if len(setYColumnList) == 2 and [0, 1] != [int(i) for i in setYColumnList]:
        newYcolumnsList = []
        for i in YColumnList:
            if i == setYColumnList[0]:
                newYcolumnsList.append(0)
            else:
                newYcolumnsList.append(1)
        df_Y = DataFrame({ColumnID: newYcolumnsList})
    logit = sm.Logit(df_Y, df_X)
    res = logit.fit()
    res_all = res.summary()
    LogLikelihood = [i.strip() for i in str(res_all).split("\n")[6].split("   ") if i][3]
    # 没找到具体参数， 只能这么分割
    index_var = [i.strip() for i in str(res_all).split("\n")[12].split("   ") if i]
    intercept = [i.strip() for i in str(res_all).split("\n")[13].split("   ") if i]
    std_err = [index_var[2], intercept[2]]
    z = [index_var[3], intercept[3]]
    P_z = [index_var[4], intercept[4]]  # 显著性
    interval_25 = [index_var[5], intercept[5]]
    interval_975 = [index_var[6], intercept[6]]
    Odds_Ratio = [math.e ** i for i in list(res.params)]
    return {
        "No_Observations": res.nobs,  # No. Observations
        "Pseudo_R": res.prsquared,  # Pseudo R^2
        "Log_Likelihood": LogLikelihood,  # LogLikelihood
        "LLNull": res.llnull,
        "llr_pvalue": res.llr_pvalue,  # ｌｌｒ显著性
        "coef": list(res.params),  # 系数
        "std_err": std_err,
        "Odds_Ratio": Odds_Ratio,
        "z": z,
        "P": P_z,  # 显著性
        "interval_25": interval_25,  # 区间0.025
        "interval_975": interval_975
    }


# -----<词云分析>----- #
def mongoWordFrequencyDetail(PlatID, mongodb_cfg, ProjID, dataSetId, ColumnIDs, dataSetIdFlag, CasesCondition=None):
    if dataSetIdFlag and 'dataSetId' in dataSetIdFlag.keys()  and dataSetId in dataSetIdFlag['dataSetId'].keys():
        dataSetFlag = dataSetIdFlag['dataSetId'][dataSetId]
        versionNum = dataSetIdFlag['version'][dataSetId]
    else:
        retVersion = dataversion(mongodb_cfg)
        dataVersion,dataSetFlag = retVersion.version(PlatID, ProjID, dataSetId)
        #retVersion.close()
    versionNum = None

    wordFreObj = mongoCorrelationAnalysisModel(mongodb_cfg)
    wordData = wordFreObj.showDatas(dataSetFlag, ProjID, dataSetId, versionNum, ColumnIDs, CasesCondition)
    #wordFreObj.close()
    string_data = "".join(["".join(i.values()) for i in wordData])

    pattern = re.compile('\t|\n|\.|-|:|;|\)|\(|\?|（|）|\|"|\u3000|,')  # 定义正则表达式匹配模式
    string_data = re.sub(pattern, '', string_data)  # 将符合模式的字符去除

    seg_list_exact = jieba.cut(string_data, cut_all=False)  # 精确模式分词
    object_list = []
    remove_words = [u'的', u'，', u'和', u'是', u'随着', u'对于', u'对', u'等', u'能', u'都', u'。', u' ', u'、',
                    u'中', u'在', u'了', u'通常', u'如果', u'我们', u'需要']  # 自定义去除词库

    for word in seg_list_exact:  # 循环读出每个分词
        if word not in remove_words:  # 如果不在去除词库中
            object_list.append(word)  # 分词追加到列表

    word_counts = collections.Counter(object_list)  # 对分词做词频统计
    word_counts_top10 = word_counts.most_common(200)  # 获取前10最高频的词
    word_data_list = []
    for k, v in word_counts_top10:
        if len(k) < 6 and len(k) > 1:
            word_data_list.append((k, v))

    return word_data_list


def is_number(s):
    try:  # 如果能运行float(s)语句，返回True（字符串s是浮点数）
        float(s)
        return True
    except ValueError:  # ValueError为Python的一种标准异常，表示"传入无效的参数"
        pass  # 如果引发了ValueError这种异常，不做任何事情（pass：不做任何事情，一般用做占位语句）
    try:
        import unicodedata  # 处理ASCii码的包
        for n in s:
            unicodedata.numeric(n)  # 把一个表示数字的字符串转换为浮点数返回的函数
            #return True
        return True
    except (TypeError, ValueError):
        pass
    return False


def is_number_int(s):
    try:  # 如果能运行int(s)语句，返回True（字符串s是浮点数）
        int(s)
        return True
    except ValueError:  # ValueError为Python的一种标准异常，表示"传入无效的参数"
        pass  # 如果引发了ValueError这种异常，不做任何事情（pass：不做任何事情，一般用做占位语句）
    return False


def check_json_format(raw_msg):
    if isinstance(raw_msg, str):
        try:
            json.loads(raw_msg)
            for key, value in json.loads(str(raw_msg).strip()).items():
                return True
        except Exception as e:
            pass
        return False
    else:
        return False


def formatHundredPercent(detailValid, point):
    percent = 0.0
    for detailValidItem in detailValid:
        percent += float(str(formatdata(detailValidItem["validpercent"], int(point) + 2)))
    percent = float(str(formatdata(percent, int(point) + 2)))
    if percent == 1.0:
        for detailValidIndex, detailValidItem in enumerate(detailValid):
            detailValid[detailValidIndex]["validpercent"] = \
                float(str(formatdata(detailValidItem["validpercent"], int(point) + 2)))
        return detailValid
    elif len(detailValid) == 0:
        return detailValid
    else:
        detailValid.sort(key=lambda x: x["validpercent"], reverse=True)
        if percent > 1.0:
            per, check = float(str(formatdata(percent - 1.0, int(point) + 2))), ["5", "6", "7", "8", "9"]
        else:
            per, check = float(str(formatdata(1.0 - percent, int(point) + 2))), ["4", "3", "2", "1"]
        for checkItem in check:
            for detailValidIndex, detailValidItem in enumerate(detailValid):
                if per == 0.0:
                    break
                else:
                    if len(str(detailValidItem["validpercent"])) >= int(point) + 5 and \
                            str(detailValidItem["validpercent"])[int(point) + 4:int(point) + 5] == checkItem:
                        if percent > 1.0:
                            detailValid[detailValidIndex]["validpercent"] = \
                                float(str(formatdata(detailValidItem["validpercent"], int(point) + 2))) - \
                                float("0." + "1".rjust(int(point) + 2,'0'))
                        else:
                            detailValid[detailValidIndex]["validpercent"] = \
                                float(str(formatdata(detailValidItem["validpercent"], int(point) + 2))) + \
                                float("0." + "1".rjust(int(point) + 2, '0'))
                        per = float(str(formatdata(per - float("0." + "1".rjust(int(point) + 2,'0')), int(point) + 2)))
            if per == 0.0:
                break
        for detailValidIndex, detailValidItem in enumerate(detailValid):
            detailValid[detailValidIndex]["validpercent"] = \
                float(str(formatdata(detailValidItem["validpercent"], int(point) + 2)))
        return detailValid


def formatdata(_data: float, _len: int, _type = None) -> object:
    formatstr, _len = '{:.2f}', int(_len)
    if _len == 0:
        text = Decimal(str(_data)).quantize(Decimal('1.'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.0f}'
    elif _len == 1:
        text = Decimal(str(_data)).quantize(Decimal('0.1'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.1f}'
    elif _len == 2:
        text = Decimal(str(_data)).quantize(Decimal('0.01'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.2f}'
    elif _len == 3:
        text = Decimal(str(_data)).quantize(Decimal('0.001'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.3f}'
    elif _len == 4:
        text = Decimal(str(_data)).quantize(Decimal('0.0001'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.4f}'
    elif _len == 5:
        text = Decimal(str(_data)).quantize(Decimal('0.00001'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.5f}'
    elif _len == 6:
        text = Decimal(str(_data)).quantize(Decimal('0.000001'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.6f}'
    elif _len == 7:
        text = Decimal(str(_data)).quantize(Decimal('0.0000001'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.7f}'
    elif _len == 8:
        text = Decimal(str(_data)).quantize(Decimal('0.00000001'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.8f}'
    elif _len == 9:
        text = Decimal(str(_data)).quantize(Decimal('0.000000001'), rounding = "ROUND_HALF_UP")
        formatstr = '{:.9f}'
    else:
        text = Decimal(str(_data)).quantize(Decimal('1.'), rounding = "ROUND_HALF_UP")
    if _type is None:
        if _len == 0:
            text = str(round(text))
        else:
            text = formatstr.format(float(text))
    elif _type == "c":
        if _len == 0:
            text = round(text)
        else:
            text = float(text)
    else:
        if _len == 0:
            text = str(round(text))
        else:
            text = str(text)
    return text


def calculation_formula(detailValid, formula):
    # 创建一个安全的解释器
    addval = Interpreter()
    for detailValidIndex, detailValidItem in enumerate(detailValid):
        if 'count' in detailValidItem.keys() and detailValidItem['count'] > 0:
            try:
                key_new = ''
                for key, value in detailValidItem.items():
                    if formula.find(key) > -1:
                        # 定义变量
                        key_new += key
                        addval.symtable[key] = value
                        # 定义表达式字符串
                expression = formula
                # 安全地计算表达式
                detailValid[detailValidIndex][key_new] = addval(expression)
            except Exception as e:
                pass


class chiSquareway:
    def __init__(self):
        pass

    @classmethod
    def run(cls, df_dropna, variableOne, variableTwo):
        try:

            setVariableOne = list(set(df_dropna[variableOne]))
            setvariableTwo = list(set(df_dropna[variableTwo]))

            myNewPandasDict = {}
            for i in setVariableOne:
                myNewPandasSubKeyDataDict = {}
                for j in range(len(df_dropna.index)):
                    if i == df_dropna.iloc[j][variableOne]:
                        myNewPandasSubKeyDataDict[df_dropna.iloc[j][variableTwo]] = df_dropna.iloc[j]["count"]
                myNewPandasDict[i] = myNewPandasSubKeyDataDict
            dataCountValueDataFrame = DataFrame(myNewPandasDict).fillna(0)
            myNewPandasList = dataCountValueDataFrame.values.tolist()
            floatNi = []  # 行的和
            floatNj = []  # 列的和
            chisq = 0

            for i in range(len(dataCountValueDataFrame.index)):
                floatNi.append(dataCountValueDataFrame.iloc[i].sum())
            for i in dataCountValueDataFrame.columns:
                floatNj.append(dataCountValueDataFrame[i].sum())

            sumTotal = sum(floatNi)
            Eij = []  # 理论频率， 二维数组

            for i in range(len(dataCountValueDataFrame.index)):

                for j in range(len(dataCountValueDataFrame.iloc[i])):
                    subValue = dataCountValueDataFrame.iloc[i].iloc[j]  # 原始值
                    subEij = floatNi[i] * floatNj[j] / sumTotal  # 理论频率
                    subChisqValue = ((subValue - subEij) * (subValue - subEij)) / float(subEij)

                    chisq += subChisqValue

            df = (len(floatNi) - 1) * (len(floatNj) - 1)
            P = stats.chi2.sf(chisq, df)
            if np.isnan(P):
                P = "NAN"
            # df 自由度
            # p 显著性
            # 值 pvalue
            # chisq 卡方
            # p 大写
            ChiSquare2wayValue = {"chisq": chisq, "df": df, "N": int(sumTotal), "P": P}
        except Exception as e:
            ChiSquare2wayValue = {"chisq": "NAN", "df": "NAN", "P": "NAN", "N": "NAN"}

        return ChiSquare2wayValue


if __name__ == '__main__':
    # print(ret)
    PlatID, mongodb_cfg = "1", "1"
    ProjID = "1"
    dataSetId = "5ee0835b23c32810e9305b5c"
    RowID="P1Q1"
    VariableID1 = "P4Q7"
    #VariableIDs = ["P1Q1","P4Q7"]
    VariableIDs = ["P4Q1A1", "P5Q1A1"]
    VariableID = "zkeyUserInfoSYXMC"

    #data=mongo_means_logic(PlatID, mongodb_cfg, ProjID, dataSetId,VariableID1,"","",{"zkeyUserInfoSYXDM": "1001100"})
    data = mongo_crosstabs_logic(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, VariableID1, "", "","", {"zkeyUserInfoSYXDM": "1001100"})
    #data = mongo_comparemeans_logic(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, VariableID1, "", {"zkeyUserInfoSYXDM": "1001100"})
    #data = mongoLinearRegressionDetail(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, VariableID1, {"zkeyUserInfoSYXDM": "1001100"})
    #data = mongoTtest_indResultDetail(PlatID, mongodb_cfg, ProjID, dataSetId, RowID, VariableID1,1,5, {"zkeyUserInfoSYXDM": "1001100"})
    #data = mongoCorrelationAnalysisDetail(PlatID, mongodb_cfg, ProjID, dataSetId, VariableIDs, "pearson", {"zkeyUserInfoSYXDM": "1001100"})
    #data = mongoWordFrequencyDetail(PlatID, mongodb_cfg, ProjID, dataSetId, VariableIDs, {"zkeyUserInfoSYXDM": "1001100"})
    for item in data:
        print(item)