#!/usr/bin/env?python2.7
#?-*-?coding:?utf-8?-*-
#coding=utf-8
#===============================================================================
#
#?????????FILE:?collect_sms_ready_snowball.py
#
#????????USAGE:?python?collect_sms_ready_snowball.py?時(shí)間參數(shù)1（YYYYMMDD）?時(shí)間參數(shù)2（YYYYMMDD）
#
#??DEscriptION:
#
#??????OPTIONS:?---
#?REQUIREMENTS:
#?????????BUGS:?---
#????????NOTES:?---
#???????AUTHOR:?penghao.qiao
#??????COMPANY:?MSXF
#??????VERSION:?1.0
#??????CREATED:?
#?????REVIEWER:
#?????REVISION:?---
#????TGT_TABLE:
#===============================================================================
import?MySQLdb
import?sys


import?os
import?string
import?datetime
import?time
sys.path.append（os.getenv（‘BDP_CONN‘））;import?aes?as?ConfigParser
from?BdpConnector?import?*
from?Calendar?import?*
from?HiveTasks?import?*
import?traceback
reload（sys）
sys.setdefaultencoding（?“utf-8“?）

config=ConfigParser.ConfigParser（）
config.read（“/home/hadoop/edw/etl/conf/etl.ini“）

tag_db?=?‘x_sell‘
tag_tab?=?‘collect_sms_ready_snowball‘
src_db=?‘fdl‘
src_tab?=?‘fdl_cust_info_chainfdl_acct_det_chainfdl_loanrpy_plan_chainfdl_loanbor_term_chain‘

hts?=?HiveTasks（tag_dbtag_tab）
dte=Calendar（datetime.datetime.today（））

#字符型起始日期
v_statbgdate?=?sys.argv[1]
#字符型結(jié)束日期
v_stateddate?=?sys.argv[2]
#日期型起始日期
v_start_date?=?datetime.datetime.strptime（v_statbgdate‘%Y%m%d‘）.date（）
#日期型結(jié)束日期
v_end_date???=?datetime.datetime.strptime（v_stateddate‘%Y%m%d‘）.date（）
#當(dāng)前時(shí)間
v_run_time?=?datetime.datetime.now（）.strftime（“%Y-%m-%d?%H:%M:%S“）
#當(dāng)前日期
v_run_date?=?datetime.datetime.now（）.strftime（“%Y%m%d“）
#etl抽數(shù)日期
v_etl_date??=?datetime.datetime.now（）.strftime（“%Y-%m-%d?%H:%M:%S“）
#T+1統(tǒng)計(jì)日期
v_yest_date?=?dte.getDayBefore（‘‘）
#前天日期
v_bf_yest_date?=?dte.getDayBeforeN（‘‘2）
#對(duì)時(shí)間參數(shù)進(jìn)行處理
if?v_statbgdate.strip（）==‘‘?and?v_stateddate.strip（）==‘‘:
????v_statbgdate=v_yest_date
????v_stateddate=v_yest_date
elif?v_statbgdate.strip（）!=‘‘?and?v_stateddate.strip（）==‘‘:
????v_stateddate=v_statbgdate
elif?v_statbgdate.strip（）==‘‘?and?v_stateddate.strip（）!=‘‘:
????v_statbgdate=v_stateddate

#定義lzo數(shù)據(jù)索引路徑
lzo_index_path=None
#定義該表數(shù)據(jù)是否要壓縮
lzo_compress?=?False
#定義是否有分區(qū)
is_partition?=?True
#定義數(shù)據(jù)文件是否要做合并
merge_flag?=?False
#合并文件路徑
merge_part_dir?=?None
#定義是否需要做表解析
parse_flag?=?False

#################Need?user?configure?the?parameters####################
v_begin_time=datetime.datetime.now（）.strftime（“%Y-%m-%d?%H:%M:%S“）
conn=MySQLdb.connect（host=config.get（“MYSQL_DB““IP“）port=int（config.get（“MYSQL_DB““PORT“））user=config.get（“MYSQL_DB““USERNAME“）passwd=config.get（“MYSQL_DB““PASSWORD“）db=config.get（“MYSQL_DB““DB“））
cur=conn.cursor（）



v_sql?=?“““
????set?hive.exec.max.dynamic.partitions=100000;
????set?hive.exec.max.dynamic.partitions.pernode=100000;
????set?fs.hdfs.impl.disable.cache?=?true;

????use?fdl;
????-------篩選出一對(duì)一，一對(duì)多（合同）數(shù)據(jù)去除和包產(chǎn)品----------
????drop?table?if?exists??tmp.tmp_collect_sms_ready_snowball_01;
??

?屬性????????????大小?????日期????時(shí)間???名稱
-----------?---------??----------?-----??----

?????文件?????????25??2018-03-01?11:35??edwscripts\.git\COMMIT_EDITMSG

?????文件????????303??2018-03-01?11:34??edwscripts\.git\config

?????文件?????????73??2018-03-01?11:34??edwscripts\.git\description

?????文件?????????23??2018-03-01?11:34??edwscripts\.git\HEAD

?????文件????????478??2018-03-01?11:34??edwscripts\.git\hooks\applypatch-msg.sample

?????文件????????896??2018-03-01?11:34??edwscripts\.git\hooks\commit-msg.sample

?????文件????????189??2018-03-01?11:34??edwscripts\.git\hooks\post-update.sample

?????文件????????424??2018-03-01?11:34??edwscripts\.git\hooks\pre-applypatch.sample

?????文件???????1642??2018-03-01?11:34??edwscripts\.git\hooks\pre-commit.sample

?????文件???????1348??2018-03-01?11:34??edwscripts\.git\hooks\pre-push.sample

?????文件???????4951??2018-03-01?11:34??edwscripts\.git\hooks\pre-rebase.sample

?????文件????????544??2018-03-01?11:34??edwscripts\.git\hooks\pre-receive.sample

?????文件???????1239??2018-03-01?11:34??edwscripts\.git\hooks\prepare-commit-msg.sample

?????文件???????3610??2018-03-01?11:34??edwscripts\.git\hooks\update.sample

?????文件?????142544??2018-03-01?11:35??edwscripts\.git\index

?????文件????????240??2018-03-01?11:34??edwscripts\.git\info\exclude

?????文件????????350??2018-03-01?11:35??edwscripts\.git\logs\HEAD

?????文件????????350??2018-03-01?11:35??edwscripts\.git\logs\refs\heads\master

?????文件????????185??2018-03-01?11:34??edwscripts\.git\logs\refs\remotes\origin\HEAD

?????文件????????147??2018-03-01?11:35??edwscripts\.git\logs\refs\remotes\origin\master

?????文件???????1929??2018-03-01?11:35??edwscripts\.git\objects\0a\59808fc38096827dac29103c76125b8aba6ce1

?????文件????????268??2018-03-01?11:35??edwscripts\.git\objects\0d\cc579cd34e95bb428606b8109b0dbabb4b7c6d

?????文件????????189??2018-03-01?11:35??edwscripts\.git\objects\94\b52d6e3986bbe54bba645f095948743aa35c72

?????文件????????165??2018-03-01?11:35??edwscripts\.git\objects\98\293b76e3866943021c198cf53a23c16cb2aa9d

?????文件????????282??2018-03-01?11:35??edwscripts\.git\objects\c8\12bc866c0e617abd85880ea70efa57a7b26c5e

?????文件????????325??2018-03-01?11:35??edwscripts\.git\objects\f7\2adcf8cd1800f7b8c97bfc62dbd2bccec29fd3

?????文件?????365688??2018-03-01?11:34??edwscripts\.git\objects\pack\pack-2fc0ed09b6b3f6d86632a410acc1b5e2a395fa73.idx

?????文件????4953896??2018-03-01?11:34??edwscripts\.git\objects\pack\pack-2fc0ed09b6b3f6d86632a410acc1b5e2a395fa73.pack

?????文件????????594??2018-03-01?11:34??edwscripts\.git\packed-refs

?????文件?????????41??2018-03-01?11:35??edwscripts\.git\refs\heads\master

............此處省略1374個(gè)文件信息