Files
stock/tushare-stock/src/job_ods_hs_finance_cashflow.py
2024-10-12 09:09:02 +08:00

286 lines
19 KiB
Python

import tushare as ts
import MySQLdb
import math
import time
from config import CONFIG_DB
from config import CONFIG_TS
db = MySQLdb.connect(CONFIG_DB['host'], CONFIG_DB['user'], CONFIG_DB['password'], CONFIG_DB['database'], charset='utf8')
pro = ts.pro_api(CONFIG_TS['secret'])
def str2float( str ):
try:
val = float(str)
if math.isinf(val) or math.isnan(val) :
val = 0
return val
except Exception as e:
return 0.0
def queryts( code ) :
df = pro.cashflow(**{
"ts_code": code,
"ann_date": "",
"f_ann_date": "",
"start_date": 20220630,
"end_date": 20230604,
#"start_date": 20170101,
#"end_date": 20220630,
"period": "",
"report_type": "",
"comp_type": "",
"end_type": "",
"is_calc": "",
"limit": "",
"offset": ""
}, fields=[
"ts_code",
"ann_date",
"f_ann_date",
"end_date",
"comp_type",
"report_type",
"end_type",
"net_profit",
"finan_exp",
"c_fr_sale_sg",
"recp_tax_rends",
"n_depos_incr_fi",
"n_incr_loans_cb",
"n_inc_borr_oth_fi",
"prem_fr_orig_contr",
"n_incr_insured_dep",
"n_reinsur_prem",
"n_incr_disp_tfa",
"ifc_cash_incr",
"n_incr_disp_faas",
"n_incr_loans_oth_bank",
"n_cap_incr_repur",
"c_fr_oth_operate_a",
"c_inf_fr_operate_a",
"c_paid_goods_s",
"c_paid_to_for_empl",
"c_paid_for_taxes",
"n_incr_clt_loan_adv",
"n_incr_dep_cbob",
"c_pay_claims_orig_inco",
"pay_handling_chrg",
"pay_comm_insur_plcy",
"oth_cash_pay_oper_act",
"st_cash_out_act",
"n_cashflow_act",
"oth_recp_ral_inv_act",
"c_disp_withdrwl_invest",
"c_recp_return_invest",
"n_recp_disp_fiolta",
"n_recp_disp_sobu",
"stot_inflows_inv_act",
"c_pay_acq_const_fiolta",
"c_paid_invest",
"n_disp_subs_oth_biz",
"oth_pay_ral_inv_act",
"n_incr_pledge_loan",
"stot_out_inv_act",
"n_cashflow_inv_act",
"c_recp_borrow",
"proc_issue_bonds",
"oth_cash_recp_ral_fnc_act",
"stot_cash_in_fnc_act",
"free_cashflow",
"c_prepay_amt_borr",
"c_pay_dist_dpcp_int_exp",
"incl_dvd_profit_paid_sc_ms",
"oth_cashpay_ral_fnc_act",
"stot_cashout_fnc_act",
"n_cash_flows_fnc_act",
"eff_fx_flu_cash",
"n_incr_cash_cash_equ",
"c_cash_equ_beg_period",
"c_cash_equ_end_period",
"c_recp_cap_contrib",
"incl_cash_rec_saims",
"uncon_invest_loss",
"prov_depr_assets",
"depr_fa_coga_dpba",
"amort_intang_assets",
"lt_amort_deferred_exp",
"decr_deferred_exp",
"incr_acc_exp",
"loss_disp_fiolta",
"loss_scr_fa",
"loss_fv_chg",
"invest_loss",
"decr_def_inc_tax_assets",
"incr_def_inc_tax_liab",
"decr_inventories",
"decr_oper_payable",
"incr_oper_payable",
"others",
"im_net_cashflow_oper_act",
"conv_debt_into_cap",
"conv_copbonds_due_within_1y",
"fa_fnc_leases",
"im_n_incr_cash_equ",
"net_dism_capital_add",
"net_cash_rece_sec",
"credit_impa_loss",
"use_right_asset_dep",
"oth_loss_asset",
"end_bal_cash",
"beg_bal_cash",
"end_bal_cash_equ",
"beg_bal_cash_equ",
"update_flag"
])
return df
def getsqlstr(row):
sqlstr = "insert into t_ods_hs_finance_cashflow(ts_code, ann_date, f_ann_date, end_date, comp_type, report_type, end_type, net_profit, finan_exp, c_fr_sale_sg, \
recp_tax_rends, n_depos_incr_fi, n_incr_loans_cb, n_inc_borr_oth_fi, prem_fr_orig_contr, n_incr_insured_dep, n_reinsur_prem, n_incr_disp_tfa, ifc_cash_incr, \
n_incr_disp_faas, n_incr_loans_oth_bank, n_cap_incr_repur, c_fr_oth_operate_a, c_inf_fr_operate_a, c_paid_goods_s, c_paid_to_for_empl, c_paid_for_taxes, \
n_incr_clt_loan_adv, n_incr_dep_cbob, c_pay_claims_orig_inco, pay_handling_chrg, pay_comm_insur_plcy, oth_cash_pay_oper_act, st_cash_out_act, n_cashflow_act, \
oth_recp_ral_inv_act, c_disp_withdrwl_invest, c_recp_return_invest, n_recp_disp_fiolta, n_recp_disp_sobu, stot_inflows_inv_act, c_pay_acq_const_fiolta, c_paid_invest, \
n_disp_subs_oth_biz, oth_pay_ral_inv_act, n_incr_pledge_loan, stot_out_inv_act, n_cashflow_inv_act, c_recp_borrow, proc_issue_bonds, oth_cash_recp_ral_fnc_act, \
stot_cash_in_fnc_act, free_cashflow, c_prepay_amt_borr, c_pay_dist_dpcp_int_exp, incl_dvd_profit_paid_sc_ms, oth_cashpay_ral_fnc_act, stot_cashout_fnc_act, \
n_cash_flows_fnc_act, eff_fx_flu_cash, n_incr_cash_cash_equ, c_cash_equ_beg_period, c_cash_equ_end_period, c_recp_cap_contrib, incl_cash_rec_saims, uncon_invest_loss, \
prov_depr_assets, depr_fa_coga_dpba, amort_intang_assets, lt_amort_deferred_exp, decr_deferred_exp, incr_acc_exp, loss_disp_fiolta, loss_scr_fa, loss_fv_chg, invest_loss, \
decr_def_inc_tax_assets, incr_def_inc_tax_liab, decr_inventories, decr_oper_payable, incr_oper_payable, others, im_net_cashflow_oper_act, conv_debt_into_cap, \
conv_copbonds_due_within_1y, fa_fnc_leases, im_n_incr_cash_equ, net_dism_capital_add, net_cash_rece_sec, credit_impa_loss, use_right_asset_dep, oth_loss_asset, \
end_bal_cash, beg_bal_cash, end_bal_cash_equ, beg_bal_cash_equ, update_flag) \
values('%s', '%s', '%s', '%s', '%s', '%s', '%s', \
%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, \
%f, %f, %f, %f, %f, %f, %f, %f, %f, '%s') \
on duplicate key update report_type='%s', comp_type='%s', end_type='%s', \
net_profit=%f, finan_exp=%f, c_fr_sale_sg=%f, recp_tax_rends=%f, n_depos_incr_fi=%f, n_incr_loans_cb=%f, n_inc_borr_oth_fi=%f, prem_fr_orig_contr=%f, \
n_incr_insured_dep=%f, n_reinsur_prem=%f, n_incr_disp_tfa=%f, ifc_cash_incr=%f, n_incr_disp_faas=%f, n_incr_loans_oth_bank=%f, n_cap_incr_repur=%f, \
c_fr_oth_operate_a=%f, c_inf_fr_operate_a=%f, c_paid_goods_s=%f, c_paid_to_for_empl=%f, c_paid_for_taxes=%f, n_incr_clt_loan_adv=%f, n_incr_dep_cbob=%f, \
c_pay_claims_orig_inco=%f, pay_handling_chrg=%f, pay_comm_insur_plcy=%f, oth_cash_pay_oper_act=%f, st_cash_out_act=%f, n_cashflow_act=%f, \
oth_recp_ral_inv_act=%f, c_disp_withdrwl_invest=%f, c_recp_return_invest=%f, n_recp_disp_fiolta=%f, n_recp_disp_sobu=%f, stot_inflows_inv_act=%f, \
c_pay_acq_const_fiolta=%f, c_paid_invest=%f, n_disp_subs_oth_biz=%f, oth_pay_ral_inv_act=%f, n_incr_pledge_loan=%f, stot_out_inv_act=%f, n_cashflow_inv_act=%f, \
c_recp_borrow=%f, proc_issue_bonds=%f, oth_cash_recp_ral_fnc_act=%f, stot_cash_in_fnc_act=%f, free_cashflow=%f, c_prepay_amt_borr=%f, c_pay_dist_dpcp_int_exp=%f, \
incl_dvd_profit_paid_sc_ms=%f, oth_cashpay_ral_fnc_act=%f, stot_cashout_fnc_act=%f, n_cash_flows_fnc_act=%f, eff_fx_flu_cash=%f, n_incr_cash_cash_equ=%f, \
c_cash_equ_beg_period=%f, c_cash_equ_end_period=%f, c_recp_cap_contrib=%f, incl_cash_rec_saims=%f, uncon_invest_loss=%f, prov_depr_assets=%f, \
depr_fa_coga_dpba=%f, amort_intang_assets=%f, lt_amort_deferred_exp=%f, decr_deferred_exp=%f, incr_acc_exp=%f, loss_disp_fiolta=%f, loss_scr_fa=%f, \
loss_fv_chg=%f, invest_loss=%f, decr_def_inc_tax_assets=%f, incr_def_inc_tax_liab=%f, decr_inventories=%f, decr_oper_payable=%f, incr_oper_payable=%f, \
others=%f, im_net_cashflow_oper_act=%f, conv_debt_into_cap=%f, conv_copbonds_due_within_1y=%f, fa_fnc_leases=%f, im_n_incr_cash_equ=%f, net_dism_capital_add=%f, \
net_cash_rece_sec=%f, credit_impa_loss=%f, use_right_asset_dep=%f, oth_loss_asset=%f, end_bal_cash=%f, beg_bal_cash=%f, end_bal_cash_equ=%f, \
beg_bal_cash_equ=%f " % (
getattr(row, 'ts_code'), getattr(row, 'ann_date'), getattr(row, 'f_ann_date'), getattr(row, 'end_date'),
getattr(row, 'report_type'), getattr(row, 'comp_type'), getattr(row, 'end_type'),
str2float(getattr(row, 'net_profit')), str2float(getattr(row, 'finan_exp')), str2float(getattr(row, 'c_fr_sale_sg')), str2float(getattr(row, 'recp_tax_rends')),
str2float(getattr(row, 'n_depos_incr_fi')), str2float(getattr(row, 'n_incr_loans_cb')), str2float(getattr(row, 'n_inc_borr_oth_fi')),
str2float(getattr(row, 'prem_fr_orig_contr')), str2float(getattr(row, 'n_incr_insured_dep')), str2float(getattr(row, 'n_reinsur_prem')),
str2float(getattr(row, 'n_incr_disp_tfa')), str2float(getattr(row, 'ifc_cash_incr')), str2float(getattr(row, 'n_incr_disp_faas')),
str2float(getattr(row, 'n_incr_loans_oth_bank')), str2float(getattr(row, 'n_cap_incr_repur')), str2float(getattr(row, 'c_fr_oth_operate_a')),
str2float(getattr(row, 'c_inf_fr_operate_a')), str2float(getattr(row, 'c_paid_goods_s')), str2float(getattr(row, 'c_paid_to_for_empl')),
str2float(getattr(row, 'c_paid_for_taxes')), str2float(getattr(row, 'n_incr_clt_loan_adv')), str2float(getattr(row, 'n_incr_dep_cbob')),
str2float(getattr(row, 'c_pay_claims_orig_inco')), str2float(getattr(row, 'pay_handling_chrg')), str2float(getattr(row, 'pay_comm_insur_plcy')),
str2float(getattr(row, 'oth_cash_pay_oper_act')), str2float(getattr(row, 'st_cash_out_act')), str2float(getattr(row, 'n_cashflow_act')),
str2float(getattr(row, 'oth_recp_ral_inv_act')), str2float(getattr(row, 'c_disp_withdrwl_invest')), str2float(getattr(row, 'c_recp_return_invest')),
str2float(getattr(row, 'n_recp_disp_fiolta')), str2float(getattr(row, 'n_recp_disp_sobu')), str2float(getattr(row, 'stot_inflows_inv_act')),
str2float(getattr(row, 'c_pay_acq_const_fiolta')), str2float(getattr(row, 'c_paid_invest')), str2float(getattr(row, 'n_disp_subs_oth_biz')),
str2float(getattr(row, 'oth_pay_ral_inv_act')), str2float(getattr(row, 'n_incr_pledge_loan')), str2float(getattr(row, 'stot_out_inv_act')),
str2float(getattr(row, 'n_cashflow_inv_act')), str2float(getattr(row, 'c_recp_borrow')), str2float(getattr(row, 'proc_issue_bonds')),
str2float(getattr(row, 'oth_cash_recp_ral_fnc_act')), str2float(getattr(row, 'stot_cash_in_fnc_act')), str2float(getattr(row, 'free_cashflow')),
str2float(getattr(row, 'c_prepay_amt_borr')), str2float(getattr(row, 'c_pay_dist_dpcp_int_exp')), str2float(getattr(row, 'incl_dvd_profit_paid_sc_ms')),
str2float(getattr(row, 'oth_cashpay_ral_fnc_act')), str2float(getattr(row, 'stot_cashout_fnc_act')), str2float(getattr(row, 'n_cash_flows_fnc_act')),
str2float(getattr(row, 'eff_fx_flu_cash')), str2float(getattr(row, 'n_incr_cash_cash_equ')), str2float(getattr(row, 'c_cash_equ_beg_period')),
str2float(getattr(row, 'c_cash_equ_end_period')), str2float(getattr(row, 'c_recp_cap_contrib')), str2float(getattr(row, 'incl_cash_rec_saims')),
str2float(getattr(row, 'uncon_invest_loss')), str2float(getattr(row, 'prov_depr_assets')), str2float(getattr(row, 'depr_fa_coga_dpba')),
str2float(getattr(row, 'amort_intang_assets')), str2float(getattr(row, 'lt_amort_deferred_exp')), str2float(getattr(row, 'decr_deferred_exp')),
str2float(getattr(row, 'incr_acc_exp')), str2float(getattr(row, 'loss_disp_fiolta')), str2float(getattr(row, 'loss_scr_fa')), str2float(getattr(row, 'loss_fv_chg')),
str2float(getattr(row, 'invest_loss')), str2float(getattr(row, 'decr_def_inc_tax_assets')), str2float(getattr(row, 'incr_def_inc_tax_liab')),
str2float(getattr(row, 'decr_inventories')), str2float(getattr(row, 'decr_oper_payable')), str2float(getattr(row, 'incr_oper_payable')),
str2float(getattr(row, 'others')), str2float(getattr(row, 'im_net_cashflow_oper_act')), str2float(getattr(row, 'conv_debt_into_cap')),
str2float(getattr(row, 'conv_copbonds_due_within_1y')), str2float(getattr(row, 'fa_fnc_leases')), str2float(getattr(row, 'im_n_incr_cash_equ')),
str2float(getattr(row, 'net_dism_capital_add')), str2float(getattr(row, 'net_cash_rece_sec')), str2float(getattr(row, 'credit_impa_loss')),
str2float(getattr(row, 'use_right_asset_dep')), str2float(getattr(row, 'oth_loss_asset')), str2float(getattr(row, 'end_bal_cash')),
str2float(getattr(row, 'beg_bal_cash')), str2float(getattr(row, 'end_bal_cash_equ')), str2float(getattr(row, 'beg_bal_cash_equ')),
getattr(row, 'update_flag'),
getattr(row, 'report_type'), getattr(row, 'comp_type'), getattr(row, 'end_type'),
str2float(getattr(row, 'net_profit')), str2float(getattr(row, 'finan_exp')), str2float(getattr(row, 'c_fr_sale_sg')), str2float(getattr(row, 'recp_tax_rends')),
str2float(getattr(row, 'n_depos_incr_fi')), str2float(getattr(row, 'n_incr_loans_cb')), str2float(getattr(row, 'n_inc_borr_oth_fi')),
str2float(getattr(row, 'prem_fr_orig_contr')), str2float(getattr(row, 'n_incr_insured_dep')), str2float(getattr(row, 'n_reinsur_prem')),
str2float(getattr(row, 'n_incr_disp_tfa')), str2float(getattr(row, 'ifc_cash_incr')), str2float(getattr(row, 'n_incr_disp_faas')),
str2float(getattr(row, 'n_incr_loans_oth_bank')), str2float(getattr(row, 'n_cap_incr_repur')), str2float(getattr(row, 'c_fr_oth_operate_a')),
str2float(getattr(row, 'c_inf_fr_operate_a')), str2float(getattr(row, 'c_paid_goods_s')), str2float(getattr(row, 'c_paid_to_for_empl')),
str2float(getattr(row, 'c_paid_for_taxes')), str2float(getattr(row, 'n_incr_clt_loan_adv')), str2float(getattr(row, 'n_incr_dep_cbob')),
str2float(getattr(row, 'c_pay_claims_orig_inco')), str2float(getattr(row, 'pay_handling_chrg')), str2float(getattr(row, 'pay_comm_insur_plcy')),
str2float(getattr(row, 'oth_cash_pay_oper_act')), str2float(getattr(row, 'st_cash_out_act')), str2float(getattr(row, 'n_cashflow_act')),
str2float(getattr(row, 'oth_recp_ral_inv_act')), str2float(getattr(row, 'c_disp_withdrwl_invest')), str2float(getattr(row, 'c_recp_return_invest')),
str2float(getattr(row, 'n_recp_disp_fiolta')), str2float(getattr(row, 'n_recp_disp_sobu')), str2float(getattr(row, 'stot_inflows_inv_act')),
str2float(getattr(row, 'c_pay_acq_const_fiolta')), str2float(getattr(row, 'c_paid_invest')), str2float(getattr(row, 'n_disp_subs_oth_biz')),
str2float(getattr(row, 'oth_pay_ral_inv_act')), str2float(getattr(row, 'n_incr_pledge_loan')), str2float(getattr(row, 'stot_out_inv_act')),
str2float(getattr(row, 'n_cashflow_inv_act')), str2float(getattr(row, 'c_recp_borrow')), str2float(getattr(row, 'proc_issue_bonds')),
str2float(getattr(row, 'oth_cash_recp_ral_fnc_act')), str2float(getattr(row, 'stot_cash_in_fnc_act')), str2float(getattr(row, 'free_cashflow')),
str2float(getattr(row, 'c_prepay_amt_borr')), str2float(getattr(row, 'c_pay_dist_dpcp_int_exp')), str2float(getattr(row, 'incl_dvd_profit_paid_sc_ms')),
str2float(getattr(row, 'oth_cashpay_ral_fnc_act')), str2float(getattr(row, 'stot_cashout_fnc_act')), str2float(getattr(row, 'n_cash_flows_fnc_act')),
str2float(getattr(row, 'eff_fx_flu_cash')), str2float(getattr(row, 'n_incr_cash_cash_equ')), str2float(getattr(row, 'c_cash_equ_beg_period')),
str2float(getattr(row, 'c_cash_equ_end_period')), str2float(getattr(row, 'c_recp_cap_contrib')), str2float(getattr(row, 'incl_cash_rec_saims')),
str2float(getattr(row, 'uncon_invest_loss')), str2float(getattr(row, 'prov_depr_assets')), str2float(getattr(row, 'depr_fa_coga_dpba')),
str2float(getattr(row, 'amort_intang_assets')), str2float(getattr(row, 'lt_amort_deferred_exp')), str2float(getattr(row, 'decr_deferred_exp')),
str2float(getattr(row, 'incr_acc_exp')), str2float(getattr(row, 'loss_disp_fiolta')), str2float(getattr(row, 'loss_scr_fa')), str2float(getattr(row, 'loss_fv_chg')),
str2float(getattr(row, 'invest_loss')), str2float(getattr(row, 'decr_def_inc_tax_assets')), str2float(getattr(row, 'incr_def_inc_tax_liab')),
str2float(getattr(row, 'decr_inventories')), str2float(getattr(row, 'decr_oper_payable')), str2float(getattr(row, 'incr_oper_payable')),
str2float(getattr(row, 'others')), str2float(getattr(row, 'im_net_cashflow_oper_act')), str2float(getattr(row, 'conv_debt_into_cap')),
str2float(getattr(row, 'conv_copbonds_due_within_1y')), str2float(getattr(row, 'fa_fnc_leases')), str2float(getattr(row, 'im_n_incr_cash_equ')),
str2float(getattr(row, 'net_dism_capital_add')), str2float(getattr(row, 'net_cash_rece_sec')), str2float(getattr(row, 'credit_impa_loss')),
str2float(getattr(row, 'use_right_asset_dep')), str2float(getattr(row, 'oth_loss_asset')), str2float(getattr(row, 'end_bal_cash')),
str2float(getattr(row, 'beg_bal_cash')), str2float(getattr(row, 'end_bal_cash_equ')), str2float(getattr(row, 'beg_bal_cash_equ'))
)
return sqlstr
def process():
stock_list = []
cursor = db.cursor()
total_stocks = 0
total_rows = 0
fd = open("./cashflow.log", 'w+')
# get stock list
sqlstr = "select ts_code from t_ods_hs_base_list where list_status = 'L' "
try:
cursor.execute(sqlstr)
results = cursor.fetchall()
for row in results :
stock_list.append(row[0])
total_stocks = total_stocks+1
except Exception as ex:
print (ex)
print ("get stock list error!")
db.close()
return
# get income data
for st_code in stock_list :
df = queryts(st_code)
loop = 0
for row in df.itertuples():
sql = getsqlstr(row)
try:
cursor.execute(sql)
db.commit()
loop = loop+1
except Exception as ex:
print (ex)
print ("get stock %s error!" % st_code )
db.rollback()
return
print("get %s cashflow data succ! total rows: %d" % (st_code, loop))
fd.write("get %s cashflow data succ! total rows: %d \n" % (st_code, loop))
fd.flush()
total_rows = total_rows+loop
time.sleep(0.5)
# get all data
print("All stock cashflow data process succ! total stocks: %d, total rows: %d " % (total_stocks, total_rows))
fd.write("All stock cashflow data process succ! total stocks: %d, total rows: %d \n" % (total_stocks, total_rows))
# close db and file
fd.close()
db.close()
process()