import tushare as ts
import MySQLdb
import math
import time

from config import CONFIG_DB 
from config import CONFIG_TS

db = MySQLdb.connect(CONFIG_DB['host'], CONFIG_DB['user'], CONFIG_DB['password'], CONFIG_DB['database'], charset='utf8')
pro = ts.pro_api(CONFIG_TS['secret'])

def str2float( str ):
    try:
        val = float(str)
        if math.isinf(val) or math.isnan(val) :
            val = 0
        return val
    except Exception as e:
        return 0.0

def queryts( code ) :
    df = pro.cashflow(**{
        "ts_code": code,
        "ann_date": "",
        "f_ann_date": "",
        "start_date": 20220630,
        "end_date": 20230604,
        #"start_date": 20170101,
        #"end_date": 20220630,
        "period": "",
        "report_type": "",
        "comp_type": "",
        "end_type": "",
        "is_calc": "",
        "limit": "",
        "offset": ""
    }, fields=[
        "ts_code",
        "ann_date",
        "f_ann_date",
        "end_date",
        "comp_type",
        "report_type",
        "end_type",
        "net_profit",
        "finan_exp",
        "c_fr_sale_sg",
        "recp_tax_rends",
        "n_depos_incr_fi",
        "n_incr_loans_cb",
        "n_inc_borr_oth_fi",
        "prem_fr_orig_contr",
        "n_incr_insured_dep",
        "n_reinsur_prem",
        "n_incr_disp_tfa",
        "ifc_cash_incr",
        "n_incr_disp_faas",
        "n_incr_loans_oth_bank",
        "n_cap_incr_repur",
        "c_fr_oth_operate_a",
        "c_inf_fr_operate_a",
        "c_paid_goods_s",
        "c_paid_to_for_empl",
        "c_paid_for_taxes",
        "n_incr_clt_loan_adv",
        "n_incr_dep_cbob",
        "c_pay_claims_orig_inco",
        "pay_handling_chrg",
        "pay_comm_insur_plcy",
        "oth_cash_pay_oper_act",
        "st_cash_out_act",
        "n_cashflow_act",
        "oth_recp_ral_inv_act",
        "c_disp_withdrwl_invest",
        "c_recp_return_invest",
        "n_recp_disp_fiolta",
        "n_recp_disp_sobu",
        "stot_inflows_inv_act",
        "c_pay_acq_const_fiolta",
        "c_paid_invest",
        "n_disp_subs_oth_biz",
        "oth_pay_ral_inv_act",
        "n_incr_pledge_loan",
        "stot_out_inv_act",
        "n_cashflow_inv_act",
        "c_recp_borrow",
        "proc_issue_bonds",
        "oth_cash_recp_ral_fnc_act",
        "stot_cash_in_fnc_act",
        "free_cashflow",
        "c_prepay_amt_borr",
        "c_pay_dist_dpcp_int_exp",
        "incl_dvd_profit_paid_sc_ms",
        "oth_cashpay_ral_fnc_act",
        "stot_cashout_fnc_act",
        "n_cash_flows_fnc_act",
        "eff_fx_flu_cash",
        "n_incr_cash_cash_equ",
        "c_cash_equ_beg_period",
        "c_cash_equ_end_period",
        "c_recp_cap_contrib",
        "incl_cash_rec_saims",
        "uncon_invest_loss",
        "prov_depr_assets",
        "depr_fa_coga_dpba",
        "amort_intang_assets",
        "lt_amort_deferred_exp",
        "decr_deferred_exp",
        "incr_acc_exp",
        "loss_disp_fiolta",
        "loss_scr_fa",
        "loss_fv_chg",
        "invest_loss",
        "decr_def_inc_tax_assets",
        "incr_def_inc_tax_liab",
        "decr_inventories",
        "decr_oper_payable",
        "incr_oper_payable",
        "others",
        "im_net_cashflow_oper_act",
        "conv_debt_into_cap",
        "conv_copbonds_due_within_1y",
        "fa_fnc_leases",
        "im_n_incr_cash_equ",
        "net_dism_capital_add",
        "net_cash_rece_sec",
        "credit_impa_loss",
        "use_right_asset_dep",
        "oth_loss_asset",
        "end_bal_cash",
        "beg_bal_cash",
        "end_bal_cash_equ",
        "beg_bal_cash_equ",
        "update_flag"
    ])
    return df

def getsqlstr(row):
    sqlstr = "insert into t_ods_hs_finance_cashflow(ts_code, ann_date, f_ann_date, end_date, comp_type, report_type, end_type, net_profit, finan_exp, c_fr_sale_sg, \
    recp_tax_rends, n_depos_incr_fi, n_incr_loans_cb, n_inc_borr_oth_fi, prem_fr_orig_contr, n_incr_insured_dep, n_reinsur_prem, n_incr_disp_tfa, ifc_cash_incr, \
    n_incr_disp_faas, n_incr_loans_oth_bank, n_cap_incr_repur, c_fr_oth_operate_a, c_inf_fr_operate_a, c_paid_goods_s, c_paid_to_for_empl, c_paid_for_taxes, \
    n_incr_clt_loan_adv, n_incr_dep_cbob, c_pay_claims_orig_inco, pay_handling_chrg, pay_comm_insur_plcy, oth_cash_pay_oper_act, st_cash_out_act, n_cashflow_act, \
    oth_recp_ral_inv_act, c_disp_withdrwl_invest, c_recp_return_invest, n_recp_disp_fiolta, n_recp_disp_sobu, stot_inflows_inv_act, c_pay_acq_const_fiolta, c_paid_invest, \
    n_disp_subs_oth_biz, oth_pay_ral_inv_act, n_incr_pledge_loan, stot_out_inv_act, n_cashflow_inv_act, c_recp_borrow, proc_issue_bonds, oth_cash_recp_ral_fnc_act, \
    stot_cash_in_fnc_act, free_cashflow, c_prepay_amt_borr, c_pay_dist_dpcp_int_exp, incl_dvd_profit_paid_sc_ms, oth_cashpay_ral_fnc_act, stot_cashout_fnc_act, \
    n_cash_flows_fnc_act, eff_fx_flu_cash, n_incr_cash_cash_equ, c_cash_equ_beg_period, c_cash_equ_end_period, c_recp_cap_contrib, incl_cash_rec_saims, uncon_invest_loss, \
    prov_depr_assets, depr_fa_coga_dpba, amort_intang_assets, lt_amort_deferred_exp, decr_deferred_exp, incr_acc_exp, loss_disp_fiolta, loss_scr_fa, loss_fv_chg, invest_loss, \
    decr_def_inc_tax_assets, incr_def_inc_tax_liab, decr_inventories, decr_oper_payable, incr_oper_payable, others, im_net_cashflow_oper_act, conv_debt_into_cap, \
    conv_copbonds_due_within_1y, fa_fnc_leases, im_n_incr_cash_equ, net_dism_capital_add, net_cash_rece_sec, credit_impa_loss, use_right_asset_dep, oth_loss_asset, \
    end_bal_cash, beg_bal_cash, end_bal_cash_equ, beg_bal_cash_equ, update_flag) \
            values('%s', '%s', '%s', '%s', '%s', '%s', '%s',  \
            %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
            %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
            %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
            %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
            %f, %f, %f, %f, %f, %f, %f, %f, %f, '%s') \
            on duplicate key update report_type='%s', comp_type='%s', end_type='%s', \
                net_profit=%f, finan_exp=%f, c_fr_sale_sg=%f, recp_tax_rends=%f, n_depos_incr_fi=%f, n_incr_loans_cb=%f, n_inc_borr_oth_fi=%f, prem_fr_orig_contr=%f, \
                n_incr_insured_dep=%f, n_reinsur_prem=%f, n_incr_disp_tfa=%f, ifc_cash_incr=%f, n_incr_disp_faas=%f, n_incr_loans_oth_bank=%f, n_cap_incr_repur=%f, \
                c_fr_oth_operate_a=%f, c_inf_fr_operate_a=%f, c_paid_goods_s=%f, c_paid_to_for_empl=%f, c_paid_for_taxes=%f, n_incr_clt_loan_adv=%f, n_incr_dep_cbob=%f, \
                c_pay_claims_orig_inco=%f, pay_handling_chrg=%f, pay_comm_insur_plcy=%f, oth_cash_pay_oper_act=%f, st_cash_out_act=%f, n_cashflow_act=%f, \
                oth_recp_ral_inv_act=%f, c_disp_withdrwl_invest=%f, c_recp_return_invest=%f, n_recp_disp_fiolta=%f, n_recp_disp_sobu=%f, stot_inflows_inv_act=%f, \
                c_pay_acq_const_fiolta=%f, c_paid_invest=%f, n_disp_subs_oth_biz=%f, oth_pay_ral_inv_act=%f, n_incr_pledge_loan=%f, stot_out_inv_act=%f, n_cashflow_inv_act=%f, \
                c_recp_borrow=%f, proc_issue_bonds=%f, oth_cash_recp_ral_fnc_act=%f, stot_cash_in_fnc_act=%f, free_cashflow=%f, c_prepay_amt_borr=%f, c_pay_dist_dpcp_int_exp=%f, \
                incl_dvd_profit_paid_sc_ms=%f, oth_cashpay_ral_fnc_act=%f, stot_cashout_fnc_act=%f, n_cash_flows_fnc_act=%f, eff_fx_flu_cash=%f, n_incr_cash_cash_equ=%f, \
                c_cash_equ_beg_period=%f, c_cash_equ_end_period=%f, c_recp_cap_contrib=%f, incl_cash_rec_saims=%f, uncon_invest_loss=%f, prov_depr_assets=%f, \
                depr_fa_coga_dpba=%f, amort_intang_assets=%f, lt_amort_deferred_exp=%f, decr_deferred_exp=%f, incr_acc_exp=%f, loss_disp_fiolta=%f, loss_scr_fa=%f, \
                loss_fv_chg=%f, invest_loss=%f, decr_def_inc_tax_assets=%f, incr_def_inc_tax_liab=%f, decr_inventories=%f, decr_oper_payable=%f, incr_oper_payable=%f, \
                others=%f, im_net_cashflow_oper_act=%f, conv_debt_into_cap=%f, conv_copbonds_due_within_1y=%f, fa_fnc_leases=%f, im_n_incr_cash_equ=%f, net_dism_capital_add=%f, \
                net_cash_rece_sec=%f, credit_impa_loss=%f, use_right_asset_dep=%f, oth_loss_asset=%f, end_bal_cash=%f, beg_bal_cash=%f, end_bal_cash_equ=%f, \
                beg_bal_cash_equ=%f " % (
                getattr(row, 'ts_code'), getattr(row, 'ann_date'), getattr(row, 'f_ann_date'), getattr(row, 'end_date'), 
                getattr(row, 'report_type'), getattr(row, 'comp_type'), getattr(row, 'end_type'), 
                str2float(getattr(row, 'net_profit')), str2float(getattr(row, 'finan_exp')), str2float(getattr(row, 'c_fr_sale_sg')), str2float(getattr(row, 'recp_tax_rends')), 
                str2float(getattr(row, 'n_depos_incr_fi')), str2float(getattr(row, 'n_incr_loans_cb')), str2float(getattr(row, 'n_inc_borr_oth_fi')), 
                str2float(getattr(row, 'prem_fr_orig_contr')), str2float(getattr(row, 'n_incr_insured_dep')), str2float(getattr(row, 'n_reinsur_prem')), 
                str2float(getattr(row, 'n_incr_disp_tfa')), str2float(getattr(row, 'ifc_cash_incr')), str2float(getattr(row, 'n_incr_disp_faas')), 
                str2float(getattr(row, 'n_incr_loans_oth_bank')), str2float(getattr(row, 'n_cap_incr_repur')), str2float(getattr(row, 'c_fr_oth_operate_a')), 
                str2float(getattr(row, 'c_inf_fr_operate_a')), str2float(getattr(row, 'c_paid_goods_s')), str2float(getattr(row, 'c_paid_to_for_empl')), 
                str2float(getattr(row, 'c_paid_for_taxes')), str2float(getattr(row, 'n_incr_clt_loan_adv')), str2float(getattr(row, 'n_incr_dep_cbob')), 
                str2float(getattr(row, 'c_pay_claims_orig_inco')), str2float(getattr(row, 'pay_handling_chrg')), str2float(getattr(row, 'pay_comm_insur_plcy')), 
                str2float(getattr(row, 'oth_cash_pay_oper_act')), str2float(getattr(row, 'st_cash_out_act')), str2float(getattr(row, 'n_cashflow_act')), 
                str2float(getattr(row, 'oth_recp_ral_inv_act')), str2float(getattr(row, 'c_disp_withdrwl_invest')), str2float(getattr(row, 'c_recp_return_invest')), 
                str2float(getattr(row, 'n_recp_disp_fiolta')), str2float(getattr(row, 'n_recp_disp_sobu')), str2float(getattr(row, 'stot_inflows_inv_act')), 
                str2float(getattr(row, 'c_pay_acq_const_fiolta')), str2float(getattr(row, 'c_paid_invest')), str2float(getattr(row, 'n_disp_subs_oth_biz')), 
                str2float(getattr(row, 'oth_pay_ral_inv_act')), str2float(getattr(row, 'n_incr_pledge_loan')), str2float(getattr(row, 'stot_out_inv_act')), 
                str2float(getattr(row, 'n_cashflow_inv_act')), str2float(getattr(row, 'c_recp_borrow')), str2float(getattr(row, 'proc_issue_bonds')), 
                str2float(getattr(row, 'oth_cash_recp_ral_fnc_act')), str2float(getattr(row, 'stot_cash_in_fnc_act')), str2float(getattr(row, 'free_cashflow')), 
                str2float(getattr(row, 'c_prepay_amt_borr')), str2float(getattr(row, 'c_pay_dist_dpcp_int_exp')), str2float(getattr(row, 'incl_dvd_profit_paid_sc_ms')), 
                str2float(getattr(row, 'oth_cashpay_ral_fnc_act')), str2float(getattr(row, 'stot_cashout_fnc_act')), str2float(getattr(row, 'n_cash_flows_fnc_act')), 
                str2float(getattr(row, 'eff_fx_flu_cash')), str2float(getattr(row, 'n_incr_cash_cash_equ')), str2float(getattr(row, 'c_cash_equ_beg_period')), 
                str2float(getattr(row, 'c_cash_equ_end_period')), str2float(getattr(row, 'c_recp_cap_contrib')), str2float(getattr(row, 'incl_cash_rec_saims')), 
                str2float(getattr(row, 'uncon_invest_loss')), str2float(getattr(row, 'prov_depr_assets')), str2float(getattr(row, 'depr_fa_coga_dpba')), 
                str2float(getattr(row, 'amort_intang_assets')), str2float(getattr(row, 'lt_amort_deferred_exp')), str2float(getattr(row, 'decr_deferred_exp')), 
                str2float(getattr(row, 'incr_acc_exp')), str2float(getattr(row, 'loss_disp_fiolta')), str2float(getattr(row, 'loss_scr_fa')), str2float(getattr(row, 'loss_fv_chg')), 
                str2float(getattr(row, 'invest_loss')), str2float(getattr(row, 'decr_def_inc_tax_assets')), str2float(getattr(row, 'incr_def_inc_tax_liab')), 
                str2float(getattr(row, 'decr_inventories')), str2float(getattr(row, 'decr_oper_payable')), str2float(getattr(row, 'incr_oper_payable')), 
                str2float(getattr(row, 'others')), str2float(getattr(row, 'im_net_cashflow_oper_act')), str2float(getattr(row, 'conv_debt_into_cap')), 
                str2float(getattr(row, 'conv_copbonds_due_within_1y')), str2float(getattr(row, 'fa_fnc_leases')), str2float(getattr(row, 'im_n_incr_cash_equ')), 
                str2float(getattr(row, 'net_dism_capital_add')), str2float(getattr(row, 'net_cash_rece_sec')), str2float(getattr(row, 'credit_impa_loss')), 
                str2float(getattr(row, 'use_right_asset_dep')), str2float(getattr(row, 'oth_loss_asset')), str2float(getattr(row, 'end_bal_cash')), 
                str2float(getattr(row, 'beg_bal_cash')), str2float(getattr(row, 'end_bal_cash_equ')), str2float(getattr(row, 'beg_bal_cash_equ')), 
                getattr(row, 'update_flag'), 
                 
                getattr(row, 'report_type'), getattr(row, 'comp_type'), getattr(row, 'end_type'), 
                str2float(getattr(row, 'net_profit')), str2float(getattr(row, 'finan_exp')), str2float(getattr(row, 'c_fr_sale_sg')), str2float(getattr(row, 'recp_tax_rends')), 
                str2float(getattr(row, 'n_depos_incr_fi')), str2float(getattr(row, 'n_incr_loans_cb')), str2float(getattr(row, 'n_inc_borr_oth_fi')), 
                str2float(getattr(row, 'prem_fr_orig_contr')), str2float(getattr(row, 'n_incr_insured_dep')), str2float(getattr(row, 'n_reinsur_prem')), 
                str2float(getattr(row, 'n_incr_disp_tfa')), str2float(getattr(row, 'ifc_cash_incr')), str2float(getattr(row, 'n_incr_disp_faas')), 
                str2float(getattr(row, 'n_incr_loans_oth_bank')), str2float(getattr(row, 'n_cap_incr_repur')), str2float(getattr(row, 'c_fr_oth_operate_a')), 
                str2float(getattr(row, 'c_inf_fr_operate_a')), str2float(getattr(row, 'c_paid_goods_s')), str2float(getattr(row, 'c_paid_to_for_empl')), 
                str2float(getattr(row, 'c_paid_for_taxes')), str2float(getattr(row, 'n_incr_clt_loan_adv')), str2float(getattr(row, 'n_incr_dep_cbob')), 
                str2float(getattr(row, 'c_pay_claims_orig_inco')), str2float(getattr(row, 'pay_handling_chrg')), str2float(getattr(row, 'pay_comm_insur_plcy')), 
                str2float(getattr(row, 'oth_cash_pay_oper_act')), str2float(getattr(row, 'st_cash_out_act')), str2float(getattr(row, 'n_cashflow_act')), 
                str2float(getattr(row, 'oth_recp_ral_inv_act')), str2float(getattr(row, 'c_disp_withdrwl_invest')), str2float(getattr(row, 'c_recp_return_invest')), 
                str2float(getattr(row, 'n_recp_disp_fiolta')), str2float(getattr(row, 'n_recp_disp_sobu')), str2float(getattr(row, 'stot_inflows_inv_act')), 
                str2float(getattr(row, 'c_pay_acq_const_fiolta')), str2float(getattr(row, 'c_paid_invest')), str2float(getattr(row, 'n_disp_subs_oth_biz')), 
                str2float(getattr(row, 'oth_pay_ral_inv_act')), str2float(getattr(row, 'n_incr_pledge_loan')), str2float(getattr(row, 'stot_out_inv_act')), 
                str2float(getattr(row, 'n_cashflow_inv_act')), str2float(getattr(row, 'c_recp_borrow')), str2float(getattr(row, 'proc_issue_bonds')), 
                str2float(getattr(row, 'oth_cash_recp_ral_fnc_act')), str2float(getattr(row, 'stot_cash_in_fnc_act')), str2float(getattr(row, 'free_cashflow')), 
                str2float(getattr(row, 'c_prepay_amt_borr')), str2float(getattr(row, 'c_pay_dist_dpcp_int_exp')), str2float(getattr(row, 'incl_dvd_profit_paid_sc_ms')), 
                str2float(getattr(row, 'oth_cashpay_ral_fnc_act')), str2float(getattr(row, 'stot_cashout_fnc_act')), str2float(getattr(row, 'n_cash_flows_fnc_act')), 
                str2float(getattr(row, 'eff_fx_flu_cash')), str2float(getattr(row, 'n_incr_cash_cash_equ')), str2float(getattr(row, 'c_cash_equ_beg_period')), 
                str2float(getattr(row, 'c_cash_equ_end_period')), str2float(getattr(row, 'c_recp_cap_contrib')), str2float(getattr(row, 'incl_cash_rec_saims')), 
                str2float(getattr(row, 'uncon_invest_loss')), str2float(getattr(row, 'prov_depr_assets')), str2float(getattr(row, 'depr_fa_coga_dpba')), 
                str2float(getattr(row, 'amort_intang_assets')), str2float(getattr(row, 'lt_amort_deferred_exp')), str2float(getattr(row, 'decr_deferred_exp')), 
                str2float(getattr(row, 'incr_acc_exp')), str2float(getattr(row, 'loss_disp_fiolta')), str2float(getattr(row, 'loss_scr_fa')), str2float(getattr(row, 'loss_fv_chg')), 
                str2float(getattr(row, 'invest_loss')), str2float(getattr(row, 'decr_def_inc_tax_assets')), str2float(getattr(row, 'incr_def_inc_tax_liab')), 
                str2float(getattr(row, 'decr_inventories')), str2float(getattr(row, 'decr_oper_payable')), str2float(getattr(row, 'incr_oper_payable')), 
                str2float(getattr(row, 'others')), str2float(getattr(row, 'im_net_cashflow_oper_act')), str2float(getattr(row, 'conv_debt_into_cap')), 
                str2float(getattr(row, 'conv_copbonds_due_within_1y')), str2float(getattr(row, 'fa_fnc_leases')), str2float(getattr(row, 'im_n_incr_cash_equ')), 
                str2float(getattr(row, 'net_dism_capital_add')), str2float(getattr(row, 'net_cash_rece_sec')), str2float(getattr(row, 'credit_impa_loss')), 
                str2float(getattr(row, 'use_right_asset_dep')), str2float(getattr(row, 'oth_loss_asset')), str2float(getattr(row, 'end_bal_cash')), 
                str2float(getattr(row, 'beg_bal_cash')), str2float(getattr(row, 'end_bal_cash_equ')), str2float(getattr(row, 'beg_bal_cash_equ')) 
            )
    return sqlstr

def process():
    stock_list = []
    cursor = db.cursor()
    total_stocks = 0
    total_rows = 0
    fd = open("./cashflow.log", 'w+')
    # get stock list
    sqlstr = "select ts_code from t_ods_hs_base_list where list_status = 'L' "
    try:
        cursor.execute(sqlstr)
        results = cursor.fetchall()
        for row in results :
            stock_list.append(row[0])
            total_stocks = total_stocks+1
    except Exception as ex:
        print (ex)
        print ("get stock list error!")
        db.close()
        return
    # get income data
    for st_code in stock_list :
        df = queryts(st_code)
        loop = 0
        for row in df.itertuples():
            sql = getsqlstr(row)
            try:
                cursor.execute(sql)
                db.commit()
                loop = loop+1
            except Exception as ex:
                print (ex)
                print ("get stock %s error!" % st_code )
                db.rollback()
                return
        print("get %s cashflow data succ! total rows: %d" % (st_code, loop))
        fd.write("get %s cashflow data succ! total rows: %d \n" % (st_code, loop))
        fd.flush()
        total_rows = total_rows+loop
        time.sleep(0.5)
    # get all data  
    print("All stock cashflow data process succ! total stocks: %d, total rows: %d " % (total_stocks, total_rows))  
    fd.write("All stock cashflow data process succ! total stocks: %d, total rows: %d \n" % (total_stocks, total_rows))   

    # close db and file
    fd.close()
    db.close()


process()